Index: tika-parsers/src/test/java/org/apache/tika/TikaTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/TikaTest.java (revision 1597125) +++ tika-parsers/src/test/java/org/apache/tika/TikaTest.java (working copy) @@ -232,7 +232,11 @@ public List getAllMetadata() { return metadatas; - } + } + + public void clear() { + metadatas.clear(); + } } } Index: tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java (revision 1597125) +++ tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java (working copy) @@ -34,6 +34,7 @@ import org.apache.tika.TikaTest; import org.apache.tika.extractor.ContainerExtractor; +import org.apache.tika.extractor.DocumentSelector; import org.apache.tika.extractor.ParserContainerExtractor; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -641,27 +642,34 @@ assertTrue(xml.contains("regressiveness")); TrackingHandler tracker = new TrackingHandler(); + RecursiveMetadataParser p = new RecursiveMetadataParser(new AutoDetectParser(), false); TikaInputStream tis = null; - ContainerExtractor ex = new ParserContainerExtractor(); + ParseContext context = new ParseContext(); + PDFParserConfig config = new PDFParserConfig(); + config.setExtractInlineImages(true); + context.set(org.apache.tika.parser.pdf.PDFParserConfig.class, config); + context.set(org.apache.tika.parser.Parser.class, p); + try { tis= TikaInputStream.get( - getResourceAsStream("/test-documents/testPDF_childAttachments.pdf")); - ex.extract(tis, ex, tracker); + getResourceAsStream("/test-documents/testPDF_childAttachments.pdf")); + p.parse(tis, new BodyContentHandler(-1), new Metadata(), context); } finally { if (tis != null) { tis.close(); } } - assertEquals(4, tracker.filenames.size()); - assertEquals(4, tracker.mediaTypes.size()); - assertNull(tracker.filenames.get(0)); - assertNull(tracker.filenames.get(1)); - assertEquals("Press Quality(1).joboptions", tracker.filenames.get(2)); - assertEquals("Unit10.doc", tracker.filenames.get(3)); - assertEquals(MediaType.image("jpeg"), tracker.mediaTypes.get(0)); - assertEquals(MediaType.image("tiff"), tracker.mediaTypes.get(1)); - assertEquals(TYPE_TEXT, tracker.mediaTypes.get(2)); - assertEquals(TYPE_DOC, tracker.mediaTypes.get(3)); + + List metadatas = p.getAllMetadata(); + assertEquals(5, metadatas.size()); + assertNull(metadatas.get(0).get(Metadata.RESOURCE_NAME_KEY)); + assertNull(metadatas.get(1).get(Metadata.RESOURCE_NAME_KEY)); + assertEquals("Press Quality(1).joboptions", metadatas.get(2).get(Metadata.RESOURCE_NAME_KEY)); + assertEquals("Unit10.doc", metadatas.get(3).get(Metadata.RESOURCE_NAME_KEY)); + assertEquals(MediaType.image("jpeg").toString(), metadatas.get(0).get(Metadata.CONTENT_TYPE)); + assertEquals(MediaType.image("tiff").toString(), metadatas.get(1).get(Metadata.CONTENT_TYPE)); + assertEquals("text/plain; charset=ISO-8859-1", metadatas.get(2).get(Metadata.CONTENT_TYPE)); + assertEquals(TYPE_DOC.toString(), metadatas.get(3).get(Metadata.CONTENT_TYPE)); } public void testVersions() throws Exception { @@ -839,4 +847,141 @@ } } } + + @Test + public void testInlineSelector() throws Exception { + + PDFParserConfig config = new PDFParserConfig(); + config.setExtractInlineImages(true); + + Parser defaultParser = new AutoDetectParser(); + + RecursiveMetadataParser p = new RecursiveMetadataParser(defaultParser, false); + ParseContext context = new ParseContext(); + context.set(org.apache.tika.parser.pdf.PDFParserConfig.class, config); + context.set(org.apache.tika.parser.Parser.class, p); + Metadata metadata = new Metadata(); + ContentHandler handler = new BodyContentHandler(-1); + String path = "/test-documents/testPDF_childAttachments.pdf"; + InputStream stream = TikaInputStream.get(this.getClass().getResource(path)); + + p.parse(stream, handler, metadata, context); + + List metadatas = p.getAllMetadata(); + int inline = 0; + int attach = 0; + for (Metadata m : metadatas) { + String v = m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (v != null) { + if (v.equals(TikaCoreProperties.EmbeddedResourceType.inline.toString())){ + inline++; + } else if (v.equals(TikaCoreProperties.EmbeddedResourceType.attachment.toString())){ + attach++; + } + } + } + assertEquals(2, inline); + assertEquals(2, attach); + + stream.close(); + p.clear(); + + //now try turning off inline + stream = TikaInputStream.get(this.getClass().getResource(path)); + + context.set(org.apache.tika.extractor.DocumentSelector.class, new AvoidInlineSelector()); + inline = 0; + attach = 0; + handler = new BodyContentHandler(-1); + metadata = new Metadata(); + p.parse(stream, handler, metadata, context); + + metadatas = p.getAllMetadata(); + for (Metadata m : metadatas) { + String v = m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (v != null) { + if (v.equals(TikaCoreProperties.EmbeddedResourceType.inline.toString())){ + inline++; + } else if (v.equals(TikaCoreProperties.EmbeddedResourceType.attachment.toString())){ + attach++; + } + } + } + assertEquals(0, inline); + assertEquals(2, attach); + + } + + + @Test + public void testInlineConfig() throws Exception { + + Parser defaultParser = new AutoDetectParser(); + RecursiveMetadataParser p = new RecursiveMetadataParser(defaultParser, false); + ParseContext context = new ParseContext(); + context.set(org.apache.tika.parser.Parser.class, p); + Metadata metadata = new Metadata(); + ContentHandler handler = new BodyContentHandler(-1); + String path = "/test-documents/testPDF_childAttachments.pdf"; + InputStream stream = TikaInputStream.get(this.getClass().getResource(path)); + + p.parse(stream, handler, metadata, context); + + List metadatas = p.getAllMetadata(); + int inline = 0; + int attach = 0; + for (Metadata m : metadatas) { + String v = m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (v != null) { + if (v.equals(TikaCoreProperties.EmbeddedResourceType.inline.toString())){ + inline++; + } else if (v.equals(TikaCoreProperties.EmbeddedResourceType.attachment.toString())){ + attach++; + } + } + } + assertEquals(0, inline); + assertEquals(2, attach); + + stream.close(); + p.clear(); + + //now try turning off inline + stream = TikaInputStream.get(this.getClass().getResource(path)); + PDFParserConfig config = new PDFParserConfig(); + config.setExtractInlineImages(true); + context.set(org.apache.tika.parser.pdf.PDFParserConfig.class, config); + inline = 0; + attach = 0; + handler = new BodyContentHandler(-1); + metadata = new Metadata(); + p.parse(stream, handler, metadata, context); + + metadatas = p.getAllMetadata(); + for (Metadata m : metadatas) { + String v = m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (v != null) { + if (v.equals(TikaCoreProperties.EmbeddedResourceType.inline.toString())){ + inline++; + } else if (v.equals(TikaCoreProperties.EmbeddedResourceType.attachment.toString())){ + attach++; + } + } + } + assertEquals(2, inline); + assertEquals(2, attach); + } + + + private class AvoidInlineSelector implements DocumentSelector { + + @Override + public boolean select(Metadata metadata) { + String v = metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (v != null && v.equals(TikaCoreProperties.EmbeddedResourceType.inline.toString())){ + return false; + } + return true; + } + } } Index: tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java (revision 1597125) +++ tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java (working copy) @@ -63,6 +63,7 @@ import org.apache.tika.io.IOExceptionWithCause; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.sax.EmbeddedContentHandler; import org.apache.tika.sax.XHTMLContentHandler; @@ -279,7 +280,7 @@ } private void extractImages(PDResources resources) throws SAXException { - if (resources == null) { + if (resources == null || config.getExtractInlineImages() == false) { return; } @@ -297,6 +298,8 @@ } else if (image instanceof PDPixelMap) { metadata.set(Metadata.CONTENT_TYPE, "image/png"); } + metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, + TikaCoreProperties.EmbeddedResourceType.inline.toString()); EmbeddedDocumentExtractor extractor = getEmbeddedDocumentExtractor(); @@ -454,6 +457,8 @@ metadata.set(Metadata.RESOURCE_NAME_KEY, ent.getKey()); metadata.set(Metadata.CONTENT_TYPE, file.getSubtype()); metadata.set(Metadata.CONTENT_LENGTH, Long.toString(file.getSize())); + metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, + TikaCoreProperties.EmbeddedResourceType.attachment.toString()); if (extractor.shouldParseEmbedded(metadata)) { TikaInputStream stream = TikaInputStream.get(file.createInputStream()); Index: tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java (revision 1597125) +++ tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java (working copy) @@ -64,6 +64,9 @@ //True if acroform content should be extracted private boolean extractAcroFormContent = true; + + //True if inline PDXImage objects should be extracted + private boolean extractInlineImages = false; //The character width-based tolerance value used to estimate where spaces in text should be added private Float averageCharTolerance; @@ -122,6 +125,9 @@ setExtractAcroFormContent( getProp(props.getProperty("extractAcroFormContent"), getExtractAcroFormContent())); + setExtractInlineImages( + getProp(props.getProperty("extractInlineImages"), + getExtractInlineImages())); } /** @@ -163,6 +169,28 @@ return extractAcroFormContent; } + /** + * If true, extract inline embedded PDXImages. + * Beware: some PDF documents of modest size (~4MB) can contain + * thousands of embedded images. At least as of with the PDFBox 1.8.5, + * there can be surprisingly large memory consumption and/or out of memory errors. + * Set to true with caution. + *

+ * The default is false. + * + * @param b + */ + public void setExtractInlineImages(boolean extractInlineImages) { + this.extractInlineImages = extractInlineImages; + + } + + /** @see #setExtractInlineImages(boolean) */ + public boolean getExtractInlineImages() { + return extractInlineImages; + } + + /** @see #setEnableAutoSpace. */ public boolean getEnableAutoSpace() { return enableAutoSpace; Index: tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties =================================================================== --- tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties (revision 1597125) +++ tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties (working copy) @@ -19,3 +19,4 @@ suppressDuplicateOverlappingText false useNonSequentialParser false extractAcroFormContent true +extractInlineImages false Index: tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java (revision 1597125) +++ tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java (working copy) @@ -36,7 +36,23 @@ */ @SuppressWarnings("deprecation") public interface TikaCoreProperties { + /** + * A file might contain different types of embedded documents. + * The most common is the attachment. + * An "inline" embedded resource should be used for embedded image + * files that are used to render the page image (as in PDXObjImages in PDF files). + *

+ * Not all parsers have yet implemented this + * + */ + public enum EmbeddedResourceType { + inline, + attachment + }; + + + /** * @see DublinCore#FORMAT */ public static final Property FORMAT = Property.composite(DublinCore.FORMAT, @@ -246,5 +262,13 @@ @Deprecated public static final Property TRANSITION_SUBJECT_TO_OO_SUBJECT = Property.composite(OfficeOpenXMLCore.SUBJECT, new Property[] { Property.internalText(Metadata.SUBJECT) }); + + /** + * See {@link #EMBEDDED_RESOURCE_TYPE} + */ + public static final Property EMBEDDED_RESOURCE_TYPE = + Property.internalClosedChoise(TikaMetadataKeys.EMBEDDED_RESOURCE_TYPE, + new String[]{EmbeddedResourceType.attachment.toString(), EmbeddedResourceType.inline.toString()}); + } Index: tika-core/src/main/java/org/apache/tika/metadata/TikaMetadataKeys.java =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/TikaMetadataKeys.java (revision 1597125) +++ tika-core/src/main/java/org/apache/tika/metadata/TikaMetadataKeys.java (working copy) @@ -26,4 +26,7 @@ String PROTECTED = "protected"; String EMBEDDED_RELATIONSHIP_ID = "embeddedRelationshipId"; + + String EMBEDDED_RESOURCE_TYPE = "embeddedResourceType"; + }