Index: tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java (revision 0) +++ tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java (revision 0) @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.net.URL; + +import junit.framework.TestCase; + +import org.apache.tika.config.TikaConfig; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.io.TikaInputStreamTest; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.Parser; +import org.apache.tika.parser.microsoft.ExcelParserTest; +import org.apache.tika.utils.ParseUtils; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Junit test class for {@link ContainerAwareDetector} + */ +public class TestContainerAwareDetector extends TestCase { + private TikaConfig tc; + private ContainerAwareDetector d; + + public void setUp() throws Exception { + tc = TikaConfig.getDefaultConfig(); + d = new ContainerAwareDetector(tc.getMimeRepository()); + } + + private InputStream getTestDoc(String filename) { + InputStream input = TestContainerAwareDetector.class.getResourceAsStream( + "/test-documents/" + filename); + assertNotNull(input); + return input; + } + + public void testDetectOLE2() throws Exception { + InputStream input; + + input = getTestDoc("testEXCEL.xls"); + assertEquals( + MediaType.application("vnd.ms-excel"), + d.detect(input, new Metadata()) + ); + + input = getTestDoc("testWORD.doc"); + assertEquals( + MediaType.application("msword"), + d.detect(input, new Metadata()) + ); + + input = getTestDoc("testPPT.ppt"); + assertEquals( + MediaType.application("vnd.ms-powerpoint"), + d.detect(input, new Metadata()) + ); + + TikaInputStream tis = TikaInputStream.get(getTestDoc("testPPT.ppt")); + assertEquals( + MediaType.application("vnd.ms-powerpoint"), + d.detect(tis, new Metadata()) + ); + assertNotNull(tis.getOpenContainer()); + } + + public void testDetectODF() throws Exception { + InputStream input; + + input = getTestDoc("testODFwithOOo3.odt"); + assertEquals( + MediaType.application("vnd.oasis.opendocument.text"), + d.detect(input, new Metadata()) + ); + + input = getTestDoc("testOpenOffice2.odf"); + assertEquals( + MediaType.application("vnd.oasis.opendocument.formula"), + d.detect(input, new Metadata()) + ); + + TikaInputStream tis = TikaInputStream.get(getTestDoc("testOpenOffice2.odf")); + assertEquals( + MediaType.application("vnd.oasis.opendocument.formula"), + d.detect(tis, new Metadata()) + ); + // Doesn't store the zip parser yet + assertNull(tis.getOpenContainer()); + } + + public void testDetectOOXML() throws Exception { + + } + + public void testDetectZip() throws Exception { + + } +} Index: tika-parsers/src/main/java/org/apache/tika/detect/ZipContainerDetector.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/detect/ZipContainerDetector.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/detect/ZipContainerDetector.java (revision 0) @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.apache.tika.io.IOUtils; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; + + +/** + * A detector that works on a Zip document + * to figure out exactly what the file is + */ +public class ZipContainerDetector implements Detector { + public MediaType detect(InputStream input, Metadata metadata) + throws IOException { + ZipInputStream zip = new ZipInputStream(input); + ZipEntry entry = zip.getNextEntry(); + while (entry != null) { + // Is it an Open Document file? + if (entry.getName().equals("mimetype")) { + String type = IOUtils.toString(zip, "UTF-8"); + int splitAt = type.indexOf('/'); + if(splitAt > -1) { + return new MediaType( + type.substring(0,splitAt), + type.substring(splitAt+1) + ); + } + return MediaType.APPLICATION_ZIP; + } else if (entry.getName().equals("[Content_Types].xml")) { + // Office Open XML + // TODO + } + entry = zip.getNextEntry(); + } + + return MediaType.APPLICATION_ZIP; + } +} + Index: tika-parsers/src/main/java/org/apache/tika/detect/ContainerAwareDetector.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/detect/ContainerAwareDetector.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/detect/ContainerAwareDetector.java (revision 0) @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.poi.poifs.common.POIFSConstants; +import org.apache.poi.poifs.storage.HeaderBlockConstants; +import org.apache.poi.util.IOUtils; +import org.apache.poi.util.LittleEndian; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; + + +/** + * A detector that knows about the container + * formats that we support (eg POIFS, Zip), + * and is able to peek inside them to + * better figure out the contents. + * Should normally be used with a + * {@link TikaInputStream} to minimise the + * memory usage. + */ +public class ContainerAwareDetector implements Detector { + private Detector fallbackDetector; + private ZipContainerDetector zipDetector; + private POIFSContainerDetector poifsDetector; + + public ContainerAwareDetector(Detector fallbackDetector) { + this.fallbackDetector = fallbackDetector; + poifsDetector = new POIFSContainerDetector(); + zipDetector = new ZipContainerDetector(); + } + + public MediaType detect(InputStream input, Metadata metadata) + throws IOException { + return detect(TikaInputStream.get(input), metadata); + } + + public MediaType detect(TikaInputStream input, Metadata metadata) + throws IOException { + + // Grab the first 8 bytes, used to do container detection + input.mark(8); + byte[] first8 = new byte[8]; + IOUtils.readFully(input, first8); + input.reset(); + + // Is this a zip file? + if(first8[0] == POIFSConstants.OOXML_FILE_HEADER[0] && + first8[1] == POIFSConstants.OOXML_FILE_HEADER[1] && + first8[2] == POIFSConstants.OOXML_FILE_HEADER[2] && + first8[3] == POIFSConstants.OOXML_FILE_HEADER[3]) { + return zipDetector.detect(input, metadata); + } + + // Is this an ole2 file? + long ole2Signature = LittleEndian.getLong(first8, 0); + if(ole2Signature == HeaderBlockConstants._signature) { + return poifsDetector.detect(input, metadata); + } + + // Not a supported container, fall back + return fallbackDetector.detect(input, metadata); + } +} + Index: tika-parsers/src/main/java/org/apache/tika/detect/POIFSContainerDetector.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/detect/POIFSContainerDetector.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/detect/POIFSContainerDetector.java (revision 0) @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.microsoft.OfficeParser; +import org.apache.tika.parser.microsoft.OfficeParser.POIFSDocumentType; + + +/** + * A detector that works on a POIFS OLE2 document + * to figure out exactly what the file is + */ +public class POIFSContainerDetector implements Detector { + public MediaType detect(InputStream input, Metadata metadata) + throws IOException { + POIFSFileSystem fs = new POIFSFileSystem(input); + + POIFSDocumentType type = + OfficeParser.POIFSDocumentType.detectType(fs); + + if(input instanceof TikaInputStream) { + ((TikaInputStream)input).setOpenContainer(fs); + } else { + fs = null; + } + + return type.getType(); + } +} + Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java (revision 958942) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java (working copy) @@ -22,7 +22,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.Locale; import java.util.Set; @@ -36,6 +35,7 @@ import org.apache.poi.poifs.filesystem.Entry; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; @@ -152,7 +152,13 @@ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); - POIFSFileSystem filesystem = new POIFSFileSystem(stream); + POIFSFileSystem filesystem; + if(stream instanceof TikaInputStream && + ((TikaInputStream)stream).getOpenContainer() != null) { + filesystem = (POIFSFileSystem)((TikaInputStream)stream).getOpenContainer(); + } else { + filesystem = new POIFSFileSystem(stream); + } // Parse summary entries first, to make metadata available early new SummaryExtractor(metadata).parseSummaries(filesystem); Index: tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java =================================================================== --- tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java (revision 958942) +++ tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java (working copy) @@ -168,6 +168,13 @@ * Marked position, or -1 if there is no current mark. */ private long mark = -1; + + /** + * A opened container, such as a POIFS FileSystem + * for an OLE2 document, or a Zip file for a + * zip based (eg ooxml, odf) document. + */ + private Object openContainer; /** * @@ -211,6 +218,26 @@ return n; } + + /** + * Returns the open container object, such as a + * POIFS FileSystem in the event of an OLE2 + * document being detected and processed by + * the OLE2 detector. + */ + public Object getOpenContainer() { + return openContainer; + } + + /** + * Stores the open container object against + * the stream, eg after a Zip contents + * detector has loaded the file to decide + * what it contains. + */ + public void setOpenContainer(Object container) { + openContainer = container; + } public File getFile() throws IOException { if (file == null) { @@ -298,6 +325,11 @@ super.mark(readlimit); mark = position; } + + @Override + public boolean markSupported() { + return true; + } @Override public void reset() throws IOException { @@ -309,6 +341,9 @@ @Override public void close() throws IOException { + if (openContainer != null) { + openContainer = null; + } if (in != null) { in.close(); in = null;