Index: tika-parsers/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java (revision 0) +++ tika-parsers/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java (revision 0) @@ -0,0 +1,200 @@ +package org.apache.tika.parser; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.sax.BasicContentHandlerFactory; +import org.apache.tika.sax.ContentHandlerFactory; +import org.junit.Test; +import org.xml.sax.helpers.DefaultHandler; + +import java.io.InputStream; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.*; + +public class RecursiveParserWrapperTest { + + @Test + public void testBasicXML() throws Exception { + List list = getMetadata(new Metadata(), + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1)); + Metadata container = list.get(0); + String content = container.get(RecursiveParserWrapper.TIKA_CONTENT); + //not much differentiates html from xml in this test file + assertTrue(content.indexOf("

") > -1); + } + + @Test + public void testBasicHTML() throws Exception { + List list = getMetadata(new Metadata(), + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.HTML, -1)); + Metadata container = list.get(0); + String content = container.get(RecursiveParserWrapper.TIKA_CONTENT); + //not much differentiates html from xml in this test file + assertTrue(content.indexOf("

") > -1); + } + + @Test + public void testBasicText() throws Exception { + List list = getMetadata(new Metadata(), + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1)); + Metadata container = list.get(0); + String content = container.get(RecursiveParserWrapper.TIKA_CONTENT); + assertTrue(content.indexOf("

-1); + } + + @Test + public void testIgnoreContent() throws Exception { + List list = getMetadata(new Metadata(), + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.IGNORE, -1)); + Metadata container = list.get(0); + String content = container.get(RecursiveParserWrapper.TIKA_CONTENT); + assertNull(content); + } + + + @Test + public void testCharLimit() throws Exception { + ParseContext context = new ParseContext(); + Metadata metadata = new Metadata(); + + Parser wrapped = new AutoDetectParser(); + RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped, + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, 60)); + InputStream stream = RecursiveParserWrapperTest.class.getResourceAsStream( + "/test-documents/test_recursive_embedded.docx"); + wrapper.parse(stream, new DefaultHandler(), metadata, context); + List list = wrapper.getMetadata(); + + assertEquals(5, list.size()); + + int wlr = 0; + for (Metadata m : list) { + String limitReached = m.get(RecursiveParserWrapper.WRITE_LIMIT_REACHED); + if (limitReached != null && limitReached.equals("true")){ + wlr++; + } + } + assertEquals(1, wlr); + + } + @Test + public void testMaxEmbedded() throws Exception { + int maxEmbedded = 4; + int totalNoLimit = 12;//including outer container file + ParseContext context = new ParseContext(); + Metadata metadata = new Metadata(); + String limitReached = null; + + Parser wrapped = new AutoDetectParser(); + RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped, + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1)); + + InputStream stream = RecursiveParserWrapperTest.class.getResourceAsStream( + "/test-documents/test_recursive_embedded.docx"); + wrapper.parse(stream, new DefaultHandler(), metadata, context); + List list = wrapper.getMetadata(); + //test default + assertEquals(totalNoLimit, list.size()); + + limitReached = list.get(0).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_LIMIT_REACHED); + assertNull(limitReached); + + + wrapper.reset(); + stream.close(); + + //test setting value + metadata = new Metadata(); + stream = RecursiveParserWrapperTest.class.getResourceAsStream( + "/test-documents/test_recursive_embedded.docx"); + wrapper.setMaxEmbeddedResources(maxEmbedded); + wrapper.parse(stream, new DefaultHandler(), metadata, context); + list = wrapper.getMetadata(); + + //add 1 for outer container file + assertEquals(maxEmbedded+1, list.size()); + + limitReached = list.get(0).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_LIMIT_REACHED); + assertEquals("true", limitReached); + + wrapper.reset(); + stream.close(); + + //test setting value < 0 + metadata = new Metadata(); + stream = RecursiveParserWrapperTest.class.getResourceAsStream( + "/test-documents/test_recursive_embedded.docx"); + + wrapper.setMaxEmbeddedResources(-2); + wrapper.parse(stream, new DefaultHandler(), metadata, context); + assertEquals(totalNoLimit, list.size()); + limitReached = list.get(0).get(RecursiveParserWrapper.EMBEDDED_RESOURCE_LIMIT_REACHED); + assertNull(limitReached); + } + + @Test + public void testEmbeddedResourcePath() throws Exception { + + Set targets = new HashSet(); + targets.add("test_recursive_embedded.docx/embed1.zip"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip/embed3.zip"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip/embed3.zip/embed4.zip"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip/embed3.zip/embed4.zip/embed4.txt"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip/embed3.zip/embed3.txt"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip/embed2a.txt"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed2.zip/embed2b.txt"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed1b.txt"); + targets.add("test_recursive_embedded.docx/embed1.zip/embed1a.txt"); + targets.add("test_recursive_embedded.docx/image1.emf"); + + Metadata metadata = new Metadata(); + metadata.set(Metadata.RESOURCE_NAME_KEY, "test_recursive_embedded.docx"); + List list = getMetadata(metadata, + new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1)); + Metadata container = list.get(0); + String content = container.get(RecursiveParserWrapper.TIKA_CONTENT); + assertTrue(content.indexOf("

") > -1); + + Set seen = new HashSet(); + for (Metadata m : list) { + String path = m.get(RecursiveParserWrapper.EMBEDDED_RESOURCE_PATH); + if (path != null) { + seen.add(path); + } + } + assertEquals(targets, seen); + } + + private List getMetadata(Metadata metadata, ContentHandlerFactory contentHandlerFactory) + throws Exception{ + ParseContext context = new ParseContext(); + Parser wrapped = new AutoDetectParser(); + RecursiveParserWrapper wrapper = new RecursiveParserWrapper(wrapped, contentHandlerFactory); + InputStream stream = RecursiveParserWrapperTest.class.getResourceAsStream( + "/test-documents/test_recursive_embedded.docx"); + wrapper.parse(stream, new DefaultHandler(), metadata, context); + return wrapper.getMetadata(); + } +} Index: tika-core/src/main/java/org/apache/tika/sax/ContentHandlerFactory.java =================================================================== --- tika-core/src/main/java/org/apache/tika/sax/ContentHandlerFactory.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/sax/ContentHandlerFactory.java (revision 0) @@ -0,0 +1,32 @@ +package org.apache.tika.sax; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.xml.sax.ContentHandler; + +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; + +/** + * Interface to allow easier injection of code for getting a new ContentHandler + */ +public interface ContentHandlerFactory { + public ContentHandler getNewContentHandler(); + public ContentHandler getNewContentHandler(OutputStream os, String encoding) throws UnsupportedEncodingException; + +} Index: tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java =================================================================== --- tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java (revision 0) @@ -0,0 +1,126 @@ +package org.apache.tika.sax; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.xml.sax.ContentHandler; +import org.xml.sax.helpers.DefaultHandler; + +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; + +/** + * Basic factory for creating common types of ContentHandlers + */ +public class BasicContentHandlerFactory implements ContentHandlerFactory { + + /** + * Common handler types for content. + */ + public enum HANDLER_TYPE { + BODY, + IGNORE, //don't store content + TEXT, + HTML, + XML + }; + + private final HANDLER_TYPE type; + private final int writeLimit; + + /** + * + * @param type basic type of handler + * @param writeLimit max number of characters to store; if < 0, the handler will store all characters + */ + public BasicContentHandlerFactory(HANDLER_TYPE type, int writeLimit) { + this.type = type; + this.writeLimit = writeLimit; + } + + @Override + public ContentHandler getNewContentHandler() { + + if (writeLimit > -1) { + switch(type) { + case BODY: + return new BodyContentHandler(writeLimit); + case IGNORE: + return new DefaultHandler(); + case TEXT: + return new WriteOutContentHandler(new ToTextContentHandler(), writeLimit); + case HTML: + return new WriteOutContentHandler(new ToHTMLContentHandler(), writeLimit); + case XML: + return new WriteOutContentHandler(new ToXMLContentHandler(), writeLimit); + default: + return new WriteOutContentHandler(new ToTextContentHandler(), writeLimit); + } + } else { + switch (type) { + case BODY: + return new BodyContentHandler(); + case IGNORE: + return new DefaultHandler(); + case TEXT: + return new ToTextContentHandler(); + case HTML: + return new ToHTMLContentHandler(); + case XML: + return new ToXMLContentHandler(); + default: + return new ToTextContentHandler(); + + } + } + } + + @Override + public ContentHandler getNewContentHandler(OutputStream os, String encoding) throws UnsupportedEncodingException { + if (writeLimit > -1) { + switch(type) { + case BODY: + return new WriteOutContentHandler(new BodyContentHandler(new ToTextContentHandler(os, encoding)), writeLimit); + case IGNORE: + return new DefaultHandler(); + case TEXT: + return new WriteOutContentHandler(new ToTextContentHandler(os, encoding), writeLimit); + case HTML: + return new WriteOutContentHandler(new ToHTMLContentHandler(os, encoding), writeLimit); + case XML: + return new WriteOutContentHandler(new ToXMLContentHandler(os, encoding), writeLimit); + default: + return new WriteOutContentHandler(new ToTextContentHandler(os, encoding), writeLimit); + } + } else { + switch (type) { + case BODY: + return new BodyContentHandler(new ToTextContentHandler(os, encoding)); + case IGNORE: + return new DefaultHandler(); + case TEXT: + return new ToTextContentHandler(os, encoding); + case HTML: + return new ToHTMLContentHandler(os, encoding); + case XML: + return new ToXMLContentHandler(os, encoding); + default: + return new ToTextContentHandler(os, encoding); + + } + } + } + +} Index: tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java (revision 0) @@ -0,0 +1,318 @@ +package org.apache.tika.parser; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.exception.TikaException; +import org.apache.tika.io.FilenameUtils; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaMetadataKeys; +import org.apache.tika.mime.MediaType; +import org.apache.tika.sax.ContentHandlerFactory; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * This is a helper class that wraps a parser in a recursive handler. + * It takes care of setting the embedded parser in the ParseContext + * and handling the embedded path calculations. + *

+ * After parsing a document, call getMetadata() to retrieve a list of + * Metadata objects, one for each embedded resource. The first item + * in the list will contain the Metadata for the outer container file. + *

+ * Content can also be extracted and stored in the {@link #TIKA_CONTENT} field + * of a Metadata object. Select the type of content to be stored + * at initialization. + *

+ * If a WriteLimitReachedException is encountered, the wrapper will stop + * processing the current resource, and it will not process + * any of the child resources for the given resource. However, it will try to + * parse as much as it can. If a WLRE is reached in the parent document, + * no child resources will be parsed. + *

+ * The implementation is based on Jukka's RecursiveMetadataParser + * and Nick's additions. See: + * RecursiveMetadataParser. + *

+ * Note that this wrapper holds all data in memory and is not appropriate + * for files with content too large to be held in memory. + *

+ * Note, too, that this wrapper is not thread safe because it stores state. + * The client must initialize a new wrapper for each thread, and the client + * is responsible for calling {@link #reset()} after each parse. + *

+ * The unit tests for this class are in the tika-parsers module. + *

+ */ +public class RecursiveParserWrapper implements Parser { + + /** + * Generated serial version + */ + private static final long serialVersionUID = 9086536568120690938L; + + + + public final static String TIKA_PREFIX = "tika:"; + public final static String TIKA_EXCEPTION_PREFIX = "tika_ex:"; + + //move this to TikaCoreProperties? + public final static Property TIKA_CONTENT = Property.internalText(TIKA_PREFIX+"content"); + public final static Property WRITE_LIMIT_REACHED = + Property.internalBoolean(TIKA_EXCEPTION_PREFIX+"write_limit_reached"); + public final static Property EMBEDDED_RESOURCE_LIMIT_REACHED = + Property.internalBoolean(TIKA_EXCEPTION_PREFIX+"embedded_resource_limit_reached"); + + //move this to TikaCoreProperties? + public final static Property EMBEDDED_RESOURCE_PATH = + Property.internalText(TIKA_PREFIX+"embedded_resource_path"); + + private final Parser wrappedParser; + private final ContentHandlerFactory contentHandlerFactory; + private final List metadatas = new LinkedList(); + + //used in naming embedded resources that don't have a name. + private int unknownCount = 0; + private int maxEmbeddedResources = -1; + private boolean hitMaxEmbeddedResources = false; + + public RecursiveParserWrapper(Parser wrappedParser, ContentHandlerFactory contentHandlerFactory) { + this.wrappedParser = wrappedParser; + this.contentHandlerFactory = contentHandlerFactory; + } + + @Override + public Set getSupportedTypes(ParseContext context) { + return wrappedParser.getSupportedTypes(context); + } + + /** + * Acts like a regular parser except it ignores the ContentHandler + * and it automatically sets/overwrites the embedded Parser in the + * ParseContext object. + *

+ * To retrieve the results of the parse, use {@link #getMetadata()}. + *

+ * Make sure to call {@link #reset()} after each parse. + */ + @Override + public void parse(InputStream stream, ContentHandler ignore, + Metadata metadata, ParseContext context) throws IOException, + SAXException, TikaException { + + String name = getResourceName(metadata); + EmbeddedParserDecorator decorator = new EmbeddedParserDecorator(name); + context.set(org.apache.tika.parser.Parser.class, decorator); + ContentHandler localHandler = contentHandlerFactory.getNewContentHandler(); + try { + wrappedParser.parse(stream, localHandler, metadata, context); + } catch (SAXException e) { + boolean wlr = isWriteLimitReached(e); + if (wlr == false) { + throw e; + } + metadata.set(WRITE_LIMIT_REACHED, "true"); + } + addContent(localHandler, metadata); + + if (hitMaxEmbeddedResources) { + metadata.set(EMBEDDED_RESOURCE_LIMIT_REACHED, "true"); + } + metadatas.add(0, deepCopy(metadata)); + } + + /** + * + * The first element in the returned list represents the + * data from the outer container file. There is no guarantee + * about the ordering of the list after that. + * + * @return list of Metadata objects that were gathered during the parse + */ + public List getMetadata() { + return metadatas; + } + + /** + * Set the maximum number of embedded resources to store. + * If the max is hit during parsing, the {@link #EMBEDDED_RESOURCE_LIMIT_REACHED} + * property will be added to the container document's Metadata. + * + *

+ * If this value is < 0 (the default), the wrapper will store all Metadata. + * + * @param max maximum number of embedded resources to store + */ + public void setMaxEmbeddedResources(int max) { + maxEmbeddedResources = max; + } + + + /** + * This clears the metadata list and resets {@link #unknownCount} and + * {@link #hitMaxEmbeddedResources} + */ + public void reset() { + metadatas.clear(); + unknownCount = 0; + hitMaxEmbeddedResources = false; + } + + /** + * Copied/modified from WriteOutContentHandler. Couldn't make that + * static, and we need to have something that will work + * with exceptions thrown from both BodyContentHandler and WriteOutContentHandler + * @param t + * @return + */ + private boolean isWriteLimitReached(Throwable t) { + if (t.getMessage().indexOf("Your document contained more than") == 0) { + return true; + } else { + return t.getCause() != null && isWriteLimitReached(t.getCause()); + } + } + + //defensive copy + private Metadata deepCopy(Metadata m) { + Metadata clone = new Metadata(); + + for (String n : m.names()){ + if (! m.isMultiValued(n)) { + clone.set(n, m.get(n)); + } else { + String[] vals = m.getValues(n); + for (int i = 0; i < vals.length; i++) { + clone.add(n, vals[i]); + } + } + } + return clone; + } + + private String getResourceName(Metadata metadata) { + String objectName = ""; + if (metadata.get(TikaMetadataKeys.RESOURCE_NAME_KEY) != null) { + objectName = metadata.get(TikaMetadataKeys.RESOURCE_NAME_KEY); + } else if (metadata.get(TikaMetadataKeys.EMBEDDED_RELATIONSHIP_ID) != null) { + objectName = metadata.get(TikaMetadataKeys.EMBEDDED_RELATIONSHIP_ID); + } else { + objectName = "embedded-" + (++unknownCount); + } + //make sure that there isn't any path info in the objectName + //some parsers can return paths, not just file names + objectName = FilenameUtils.getName(objectName); + return objectName; + } + + private void addContent(ContentHandler handler, Metadata metadata) { + + if (handler.getClass().equals(DefaultHandler.class)){ + //no-op: we can't rely on just testing for + //empty content because DefaultHandler's toString() + //returns e.g. "org.xml.sax.helpers.DefaultHandler@6c8b1edd" + } else { + String content = handler.toString(); + if (content != null && content.trim().length() > 0 ) { + metadata.add(TIKA_CONTENT, content); + } + } + + } + + /** + * Override for different behavior. + * + * @return handler to be used for each document + */ + + + private class EmbeddedParserDecorator extends ParserDecorator { + + private static final long serialVersionUID = 207648200464263337L; + + private String location = null; + + + private EmbeddedParserDecorator(String location) { + super(wrappedParser); + this.location = location; + if (! this.location.endsWith("/")) { + this.location += "/"; + } + } + + @Override + public void parse(InputStream stream, ContentHandler ignore, + Metadata metadata, ParseContext context) throws IOException, + SAXException, TikaException { + //Test to see if we should avoid parsing + if (maxEmbeddedResources > -1 && + metadatas.size() >= maxEmbeddedResources) { + hitMaxEmbeddedResources = true; + return; + } + // Work out what this thing is + String objectName = getResourceName(metadata); + String objectLocation = this.location + objectName; + + metadata.add(EMBEDDED_RESOURCE_PATH, objectLocation); + + //ignore the content handler that is passed in + //and get a fresh handler + ContentHandler localHandler = contentHandlerFactory.getNewContentHandler(); + + Parser preContextParser = context.get(Parser.class); + context.set(Parser.class, new EmbeddedParserDecorator(objectLocation)); + + try { + super.parse(stream, localHandler, metadata, context); + } catch (SAXException e) { + boolean wlr = isWriteLimitReached(e); + if (wlr == true) { + metadata.add(WRITE_LIMIT_REACHED, "true"); + } else { + throw e; + } + } finally { + context.set(Parser.class, preContextParser); + } + + //Because of recursion, we need + //to re-test to make sure that we limit the + //number of stored resources + if (maxEmbeddedResources > -1 && + metadatas.size() >= maxEmbeddedResources) { + hitMaxEmbeddedResources = true; + return; + } + addContent(localHandler, metadata); + metadatas.add(deepCopy(metadata)); + } + } + + +}