Index: tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java (revision 1085429) +++ tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java (working copy) @@ -21,7 +21,7 @@ import junit.framework.TestCase; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -70,7 +70,7 @@ assertEquals( MediaType.parse("application/vnd.ms-powerpoint"), detector.detect(stream, new Metadata())); - assertTrue(stream.getOpenContainer() instanceof POIFSFileSystem); + assertTrue(stream.getOpenContainer() instanceof NPOIFSFileSystem); } finally { stream.close(); } Index: tika-parsers/src/test/java/org/apache/tika/parser/microsoft/TNEFParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/microsoft/TNEFParserTest.java (revision 1085429) +++ tika-parsers/src/test/java/org/apache/tika/parser/microsoft/TNEFParserTest.java (working copy) @@ -17,10 +17,15 @@ package org.apache.tika.parser.microsoft; import org.apache.tika.detect.ContainerAwareDetector; +import org.apache.tika.extractor.ContainerExtractor; +import org.apache.tika.extractor.ParserContainerExtractor; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MimeTypes; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; +import org.xml.sax.ContentHandler; /** * Tests for the TNEF (winmail.dat) parser @@ -41,4 +46,49 @@ stream.close(); } } + + public void testMetadata() throws Exception { + TikaInputStream stream = getTestFile(file); + + Metadata metadata = new Metadata(); + ContentHandler handler = new BodyContentHandler(); + + TNEFParser tnef = new TNEFParser(); + tnef.parse(stream, handler, metadata, new ParseContext()); + + assertEquals("This is a test message", metadata.get(Metadata.SUBJECT)); + } + + /** + * Check the Rtf and Attachments are returned + * as expected + */ + public void testBodyAndAttachments() throws Exception { + ContainerExtractor extractor = new ParserContainerExtractor(); + + // Process it with recursing + // Will have the message body RTF and the attachments + TrackingHandler handler = process(file, extractor, true); + assertEquals(6, handler.filenames.size()); + assertEquals(6, handler.mediaTypes.size()); + + // We know the filenames for all of them + assertEquals("message.rtf", handler.filenames.get(0)); + assertEquals(MediaType.application("rtf"), handler.mediaTypes.get(0)); + + assertEquals("quick.doc", handler.filenames.get(1)); + assertEquals(MediaType.application("msword"), handler.mediaTypes.get(1)); + + assertEquals("quick.html", handler.filenames.get(2)); + assertEquals(MediaType.text("html"), handler.mediaTypes.get(2)); + + assertEquals("quick.pdf", handler.filenames.get(3)); + assertEquals(MediaType.application("pdf"), handler.mediaTypes.get(3)); + + assertEquals("quick.txt", handler.filenames.get(4)); + assertEquals(MediaType.text("plain"), handler.mediaTypes.get(4)); + + assertEquals("quick.xml", handler.filenames.get(5)); + assertEquals(MediaType.application("xml"), handler.mediaTypes.get(5)); + } } Index: tika-parsers/src/main/java/org/apache/tika/detect/POIFSContainerDetector.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/detect/POIFSContainerDetector.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/detect/POIFSContainerDetector.java (working copy) @@ -28,7 +28,7 @@ import java.util.Set; import org.apache.poi.poifs.filesystem.Entry; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.io.CloseShieldInputStream; import org.apache.tika.io.TaggedInputStream; import org.apache.tika.io.TikaInputStream; @@ -141,9 +141,13 @@ TaggedInputStream tagged = new TaggedInputStream( new BufferedInputStream(new FileInputStream(file))); try { - // POIFSFileSystem might try close the stream - POIFSFileSystem fs = - new POIFSFileSystem(new CloseShieldInputStream(tagged)); + NPOIFSFileSystem fs; + if (stream.hasFile()) { + fs = new NPOIFSFileSystem(stream.getFile()); + } else { + // Load from a stream, but prevent the stream being closed + fs = new NPOIFSFileSystem(new CloseShieldInputStream(tagged)); + } // Optimize a possible later parsing process by keeping // a reference to the already opened POI file system Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java (working copy) @@ -40,7 +40,7 @@ import org.apache.poi.hwpf.usermodel.TableRow; import org.apache.poi.poifs.filesystem.DirectoryEntry; import org.apache.poi.poifs.filesystem.Entry; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.parser.ParseContext; @@ -54,11 +54,11 @@ } protected void parse( - POIFSFileSystem filesystem, XHTMLContentHandler xhtml) + NPOIFSFileSystem filesystem, XHTMLContentHandler xhtml) throws IOException, SAXException, TikaException { HWPFDocument document; try { - document = new HWPFDocument(filesystem); + document = new HWPFDocument(filesystem.getRoot()); } catch(OldWordFileFormatException e) { parseWord6(filesystem, xhtml); return; @@ -345,9 +345,9 @@ } protected void parseWord6( - POIFSFileSystem filesystem, XHTMLContentHandler xhtml) + NPOIFSFileSystem filesystem, XHTMLContentHandler xhtml) throws IOException, SAXException, TikaException { - HWPFOldDocument doc = new HWPFOldDocument(filesystem); + HWPFOldDocument doc = new HWPFOldDocument(filesystem.getRoot()); Word6Extractor extractor = new Word6Extractor(doc); for(String p : extractor.getParagraphText()) { Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java (working copy) @@ -16,25 +16,25 @@ */ package org.apache.tika.parser.microsoft; +import java.io.IOException; +import java.util.List; + import org.apache.poi.hslf.extractor.PowerPointExtractor; import org.apache.poi.hslf.model.OLEShape; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.parser.ParseContext; import org.apache.tika.sax.XHTMLContentHandler; import org.xml.sax.SAXException; -import java.io.IOException; -import java.util.List; - public class HSLFExtractor extends AbstractPOIFSExtractor { public HSLFExtractor(ParseContext context) { super(context); } protected void parse( - POIFSFileSystem filesystem, XHTMLContentHandler xhtml) + NPOIFSFileSystem filesystem, XHTMLContentHandler xhtml) throws IOException, SAXException, TikaException { PowerPointExtractor powerPointExtractor = new PowerPointExtractor(filesystem); Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java (working copy) @@ -19,7 +19,11 @@ import java.io.IOException; import java.io.InputStream; import java.security.GeneralSecurityException; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; import org.apache.poi.hdgf.extractor.VisioTextExtractor; import org.apache.poi.hpbf.extractor.PublisherTextExtractor; @@ -27,6 +31,7 @@ import org.apache.poi.poifs.crypt.EncryptionInfo; import org.apache.poi.poifs.filesystem.DirectoryEntry; import org.apache.poi.poifs.filesystem.Entry; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; @@ -93,6 +98,10 @@ return detectType(fs.getRoot()); } + public static POIFSDocumentType detectType(NPOIFSFileSystem fs) { + return detectType(fs.getRoot()); + } + public static POIFSDocumentType detectType(DirectoryEntry node) { for (Entry entry : node) { POIFSDocumentType type = detectType(entry); @@ -152,12 +161,18 @@ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); - POIFSFileSystem filesystem; - if(stream instanceof TikaInputStream && - ((TikaInputStream)stream).getOpenContainer() != null) { - filesystem = (POIFSFileSystem)((TikaInputStream)stream).getOpenContainer(); + NPOIFSFileSystem filesystem; + if(stream instanceof TikaInputStream) { + TikaInputStream tstream = (TikaInputStream)stream; + if(tstream.getOpenContainer() != null) { + filesystem = (NPOIFSFileSystem)tstream.getOpenContainer(); + } else if(tstream.hasFile()) { + filesystem = new NPOIFSFileSystem(tstream.getFile()); + } else { + filesystem = new NPOIFSFileSystem(tstream); + } } else { - filesystem = new POIFSFileSystem(stream); + filesystem = new NPOIFSFileSystem(stream); } // Parse summary entries first, to make metadata available early Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java (working copy) @@ -21,7 +21,7 @@ import org.apache.poi.hsmf.MAPIMessage; import org.apache.poi.hsmf.datatypes.AttachmentChunks; import org.apache.poi.hsmf.exceptions.ChunkNotFoundException; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -35,11 +35,11 @@ public class OutlookExtractor extends AbstractPOIFSExtractor { private final MAPIMessage msg; - public OutlookExtractor(POIFSFileSystem filesystem, ParseContext context) throws TikaException { + public OutlookExtractor(NPOIFSFileSystem filesystem, ParseContext context) throws TikaException { super(context); try { - this.msg = new MAPIMessage(filesystem); + this.msg = new MAPIMessage(filesystem.getRoot()); } catch (IOException e) { throw new TikaException("Failed to parse Outlook message", e); } Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java (working copy) @@ -29,7 +29,7 @@ import org.apache.poi.hpsf.UnexpectedPropertySetTypeException; import org.apache.poi.poifs.filesystem.DocumentEntry; import org.apache.poi.poifs.filesystem.DocumentInputStream; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.PagedText; @@ -52,14 +52,14 @@ this.metadata = metadata; } - public void parseSummaries(POIFSFileSystem filesystem) + public void parseSummaries(NPOIFSFileSystem filesystem) throws IOException, TikaException { parseSummaryEntryIfExists(filesystem, SUMMARY_INFORMATION); parseSummaryEntryIfExists(filesystem, DOCUMENT_SUMMARY_INFORMATION); } private void parseSummaryEntryIfExists( - POIFSFileSystem filesystem, String entryName) + NPOIFSFileSystem filesystem, String entryName) throws IOException, TikaException { try { DocumentEntry entry = Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/TNEFParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/TNEFParser.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/TNEFParser.java (revision 0) @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.apache.poi.hmef.Attachment; +import org.apache.poi.hmef.HMEFMessage; +import org.apache.poi.hmef.attribute.MAPIAttribute; +import org.apache.poi.hmef.attribute.MAPIRtfAttribute; +import org.apache.poi.hsmf.datatypes.MAPIProperty; +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.sax.EmbeddedContentHandler; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +/** + * A POI-powered Tika Parser for TNEF (Transport Neutral + * Encoding Format) messages, aka winmail.dat + */ +public class TNEFParser implements Parser { + private static final long serialVersionUID = 4611820730372823452L; + + private static final Set SUPPORTED_TYPES = + Collections.unmodifiableSet(new HashSet(Arrays.asList( + MediaType.application("vnd.ms-tnef"), + MediaType.application("ms-tnef"), + MediaType.application("x-tnef") + ))); + + public Set getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + /** + * Extracts properties and text from an MS Document input stream + */ + public void parse( + InputStream stream, ContentHandler handler, + Metadata metadata, ParseContext context) + throws IOException, SAXException, TikaException { + + // We work by recursing, so get the appropriate bits + EmbeddedDocumentExtractor ex = context.get(EmbeddedDocumentExtractor.class); + EmbeddedDocumentExtractor embeddedExtractor; + if (ex==null) { + embeddedExtractor = new ParsingEmbeddedDocumentExtractor(context); + } else { + embeddedExtractor = ex; + } + + // Ask POI to process the file for us + HMEFMessage msg = new HMEFMessage(stream); + + // Set the message subject if known + String subject = msg.getSubject(); + if(subject != null && subject.length() > 0) { + metadata.set(Metadata.SUBJECT, subject); + } + + // Recurse into the message body RTF + MAPIAttribute attr = msg.getMessageMAPIAttribute(MAPIProperty.RTF_COMPRESSED); + if(attr != null && attr instanceof MAPIRtfAttribute) { + MAPIRtfAttribute rtf = (MAPIRtfAttribute)attr; + handleEmbedded( + "message.rtf", "application/rtf", + rtf.getData(), + embeddedExtractor, handler + ); + } + + // Recurse into each attachment in turn + for(Attachment attachment : msg.getAttachments()) { + String name = attachment.getLongFilename(); + if(name == null || name.length() == 0) { + name = attachment.getFilename(); + } + if(name == null || name.length() == 0) { + String ext = attachment.getExtension(); + if(ext != null) { + name = "unknown" + ext; + } + } + handleEmbedded( + name, null, attachment.getContents(), + embeddedExtractor, handler + ); + } + } + + private void handleEmbedded(String name, String type, byte[] contents, + EmbeddedDocumentExtractor embeddedExtractor, ContentHandler handler) + throws IOException, SAXException, TikaException { + Metadata metadata = new Metadata(); + if(name != null) + metadata.set(Metadata.RESOURCE_NAME_KEY, name); + if(type != null) + metadata.set(Metadata.CONTENT_TYPE, type); + + if (embeddedExtractor.shouldParseEmbedded(metadata)) { + embeddedExtractor.parseEmbedded( + TikaInputStream.get(contents), + new EmbeddedContentHandler(handler), + metadata, false); + } + } + + /** + * @deprecated This method will be removed in Apache Tika 1.0. + */ + public void parse( + InputStream stream, ContentHandler handler, Metadata metadata) + throws IOException, SAXException, TikaException { + parse(stream, handler, metadata, new ParseContext()); + } +} Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java (revision 1085429) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ExcelExtractor.java (working copy) @@ -59,7 +59,7 @@ import org.apache.poi.poifs.filesystem.DirectoryEntry; import org.apache.poi.poifs.filesystem.DocumentInputStream; import org.apache.poi.poifs.filesystem.Entry; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.parser.ParseContext; @@ -130,7 +130,7 @@ * or writing the extracted content */ protected void parse( - POIFSFileSystem filesystem, XHTMLContentHandler xhtml, + NPOIFSFileSystem filesystem, XHTMLContentHandler xhtml, Locale locale) throws IOException, SAXException, TikaException { TikaHSSFListener listener = new TikaHSSFListener(xhtml, locale, this); listener.processFile(filesystem, isListenForAllRecords()); @@ -243,7 +243,7 @@ * @throws IOException on any IO errors. * @throws SAXException on any SAX parsing errors. */ - public void processFile(POIFSFileSystem filesystem, boolean listenForAllRecords) + public void processFile(NPOIFSFileSystem filesystem, boolean listenForAllRecords) throws IOException, SAXException, TikaException { // Set up listener and register the records we want to process Index: tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser =================================================================== --- tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser (revision 1085429) +++ tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser (working copy) @@ -29,6 +29,7 @@ org.apache.tika.parser.mail.RFC822Parser org.apache.tika.parser.mbox.MboxParser org.apache.tika.parser.microsoft.OfficeParser +org.apache.tika.parser.microsoft.TNEFParser org.apache.tika.parser.microsoft.ooxml.OOXMLParser org.apache.tika.parser.mp3.Mp3Parser org.apache.tika.parser.hdf.HDFParser Index: tika-parsers/pom.xml =================================================================== --- tika-parsers/pom.xml (revision 1085429) +++ tika-parsers/pom.xml (working copy) @@ -35,7 +35,7 @@ http://tika.apache.org/ - 3.8-beta1 + 3.8-beta2