Index: tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java (revision 1577107) +++ tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java (working copy) @@ -19,22 +19,47 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.tika.Tika; import org.apache.tika.TikaTest; +import org.apache.tika.config.TikaConfig; +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.ContainerExtractor; +import org.apache.tika.extractor.EmbeddedResourceHandler; +import org.apache.tika.extractor.ParserContainerExtractor; +import org.apache.tika.io.FilenameUtils; +import org.apache.tika.io.IOUtils; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; import org.apache.tika.metadata.OfficeOpenXMLCore; +import org.apache.tika.metadata.RTFMetadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.MimeType; +import org.apache.tika.mime.MimeTypeException; +import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.parser.ParserDecorator; +import org.apache.tika.sax.BodyContentHandler; import org.apache.tika.sax.WriteOutContentHandler; import org.junit.Test; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; /** * Junit test class for the Tika {@link RTFParser} @@ -356,7 +381,24 @@ // TIKA-782 @Test public void testBinControlWord() throws Exception { - assertTrue(getXML("testBinControlWord.rtf").xml.indexOf("\u00ff\u00ff\u00ff\u00ff") == -1); + ByteCopyingHandler embHandler = new ByteCopyingHandler(); + TikaInputStream tis = null; + try { + ContainerExtractor ex = new ParserContainerExtractor(); + tis = TikaInputStream.get(getResourceAsStream("/test-documents/testBinControlWord.rtf")); + assertEquals(true, ex.isSupported(tis)); + ex.extract(tis, ex, embHandler); + } finally { + tis.close(); + } + assertEquals(1, embHandler.bytes.size()); + + byte[] bytes = embHandler.bytes.get(0); + assertEquals(10, bytes.length); + //} + assertEquals(125, (int)bytes[4]); + //make sure that at least the last value is correct + assertEquals(-1, (int)bytes[9]); } // TIKA-999 @@ -377,6 +419,146 @@ assertContains("Body", content); } + //TIKA-1010 + @Test + public void testEmbeddedMonster() throws Exception { + List trueNames = new ArrayList(); + trueNames.add("file0.doc"); + trueNames.add("Hw.txt"); + trueNames.add("file1.xlsx"); + trueNames.add("test-zip-of-zip_\u666E\u6797\u65AF\u987F.zip"); + trueNames.add("html-within-zip.zip"); + trueNames.add("text.html"); + trueNames.add("testHTML_utf8_\u666E\u6797\u65AF\u987F.html"); + trueNames.add("testJPEG_\u666E\u6797\u65AF\u987F.jpg"); + trueNames.add("file2.xls"); + trueNames.add("testMSG_\u666E\u6797\u65AF\u987F.msg"); + trueNames.add("file3.pdf"); + trueNames.add("file4.ppt"); + trueNames.add("file5.pptx"); + trueNames.add("thumbnail_0.jpeg"); + trueNames.add("file6.doc"); + trueNames.add("file7.doc"); + trueNames.add("file8.docx"); + + List trueTypes = new ArrayList(); + trueTypes.add("application/msword"); + trueTypes.add("text/plain"); + trueTypes.add("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + trueTypes.add("application/zip"); + trueTypes.add("application/zip"); + trueTypes.add("text/html"); + trueTypes.add("text/html"); + trueTypes.add("image/jpeg"); + trueTypes.add("application/vnd.ms-excel"); + trueTypes.add("application/vnd.ms-outlook"); + trueTypes.add("application/pdf"); + trueTypes.add("application/vnd.ms-powerpoint"); + trueTypes.add("application/vnd.openxmlformats-officedocument.presentationml.presentation"); + trueTypes.add("image/jpeg"); + trueTypes.add("application/msword"); + trueTypes.add("application/msword"); + trueTypes.add("application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + + TrackingHandler tracker = new TrackingHandler(true); + TikaInputStream tis = null; + try { + ContainerExtractor ex = new ParserContainerExtractor(); + tis = TikaInputStream.get(getResourceAsStream("/test-documents/testRTFEmbeddedFiles.rtf")); + assertEquals(true, ex.isSupported(tis)); + ex.extract(tis, ex, tracker); + + } finally { + tis.close(); + } + assertEquals(trueNames.size(), tracker.filenames.size()); + assertEquals(trueTypes.size(), tracker.filenames.size()); + for (int i = 0; i < tracker.filenames.size(); i++) { + assertEquals(trueNames.get(i), tracker.filenames.get(i)); + assertEquals(trueTypes.get(i), tracker.mediaTypes.get(i).toString()); + } + + tracker = new TrackingHandler(false); + tis = null; + try { + ContainerExtractor ex = new ParserContainerExtractor(); + tis = TikaInputStream.get(getResourceAsStream("/test-documents/testRTFEmbeddedFiles.rtf")); + assertEquals(true, ex.isSupported(tis)); + ex.extract(tis, ex, tracker); + + } finally { + tis.close(); + } + assertEquals(44, tracker.filenames.size()); + } + + + + //TIKA-1010 test regular (not "embedded") images/picts + + public void testRegularImages() throws Exception { + Parser base = new AutoDetectParser(); + ParseContext ctx = new ParseContext(); + RecursiveMetadataParser parser = new RecursiveMetadataParser(base); + ctx.set(org.apache.tika.parser.Parser.class, parser); + TikaInputStream tis = null; + ContentHandler handler = new BodyContentHandler(); + Metadata rootMetadata = new Metadata(); + rootMetadata.add(Metadata.RESOURCE_NAME_KEY, "testRTFRegularImages.rtf"); + try { + tis = TikaInputStream.get(getResourceAsStream("/test-documents/testRTFRegularImages.rtf")); + parser.parse(tis, handler, rootMetadata, ctx); + } finally { + tis.close(); + } + Map metadatas = parser.getAllMetadata(); + + Metadata meta_jpg_exif = metadatas.get("testJPEG_EXIF_\u666E\u6797\u65AF\u987F.jpg"); + Metadata meta_jpg = metadatas.get("testJPEG_\u666E\u6797\u65AF\u987F.jpg"); + + assertTrue(meta_jpg_exif != null); + assertTrue(meta_jpg != null); + assertTrue(Arrays.asList(meta_jpg_exif.getValues("dc:subject")).contains("serbor")); + assertTrue(meta_jpg.get("Comments").contains("Licensed to the Apache")); + //make sure old metadata doesn't linger between objects + assertFalse(Arrays.asList(meta_jpg.getValues("dc:subject")).contains("serbor")); + assertEquals("false", meta_jpg.get(RTFMetadata.PICT_IN_OBJECT)); + assertEquals("false", meta_jpg_exif.get(RTFMetadata.PICT_IN_OBJECT)); + } + + //TIKA-1010 test linked embedded doc + @Test + public void testEmbeddedLinkedDocument() throws Exception { + + TrackingHandler tracker = new TrackingHandler(true); + TikaInputStream tis = null; + try { + ContainerExtractor ex = new ParserContainerExtractor(); + tis = TikaInputStream.get(getResourceAsStream("/test-documents/testRTFEmbeddedLink.rtf")); + assertEquals(true, ex.isSupported(tis)); + ex.extract(tis, ex, tracker); + + } finally { + tis.close(); + } + //should gracefully skip link and not throw NPE, IOEx, etc + assertEquals(0, tracker.filenames.size()); + + tracker = new TrackingHandler(false); + tis = null; + try { + ContainerExtractor ex = new ParserContainerExtractor(); + tis = TikaInputStream.get(getResourceAsStream("/test-documents/testRTFEmbeddedLink.rtf")); + assertEquals(true, ex.isSupported(tis)); + ex.extract(tis, ex, tracker); + + } finally { + tis.close(); + } + //should gracefully skip link and not throw NPE, IOEx, etc + assertEquals(2, tracker.filenames.size()); + } + private Result getResult(String filename) throws Exception { File file = getResourceAsFile("/test-documents/" + filename); @@ -394,4 +576,104 @@ private String getText(String filename) throws Exception { return getResult(filename).text; } + + public static class ByteCopyingHandler implements EmbeddedResourceHandler { + + public List bytes = new ArrayList(); + + @Override + public void handle(String filename, MediaType mediaType, + InputStream stream) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + if (! stream.markSupported()) { + stream = TikaInputStream.get(stream); + } + stream.mark(0); + try { + IOUtils.copy(stream, os); + bytes.add(os.toByteArray()); + stream.reset(); + } catch (IOException e) { + //swallow + } + } + } + + public static class TrackingHandler implements EmbeddedResourceHandler { + public List filenames = new ArrayList(); + public List mediaTypes = new ArrayList(); + private final MediaType EMF = MediaType.parse("application/x-emf"); + private final MediaType MSMETA = MediaType.parse("application/x-msmetafile"); + private boolean skipEMFAndMSMeta = false; + private int count = 0; + public TrackingHandler(boolean skipEMFAndMSMeta) { + this.skipEMFAndMSMeta = skipEMFAndMSMeta; + } + + @Override + public void handle(String filename, MediaType mediaType, + InputStream stream) { + if (skipEMFAndMSMeta && (mediaType.equals(EMF) || mediaType.equals(MSMETA))){ + return; + } + mediaTypes.add(mediaType); + + MimeType mime; + try { + mime = TikaConfig.getDefaultConfig().getMimeRepository().forName(mediaType.toString()); + String actualFileName = (filename == null ) ? "file"+count++ + mime.getExtension() : filename; + //necessary to strip out the directory for Hw.txt in linux + filenames.add(FilenameUtils.getSafeFilename(actualFileName)); + } catch (MimeTypeException e1) { + filenames.add(null); + } + } + } + /** dangerous in that this relies on unique file names! */ + public static class RecursiveMetadataParser extends ParserDecorator{ + private static final long serialVersionUID = 1L; + private Map metadatas = new HashMap(); + int i = 0; + + public RecursiveMetadataParser(Parser parser) { + super(parser); + } + + @Override + public void parse( + InputStream stream, ContentHandler contentHandler, + Metadata metadata, ParseContext context) + throws IOException, SAXException, TikaException { + + super.parse(stream, contentHandler, metadata, context); + + String fName = metadata.get(Metadata.RESOURCE_NAME_KEY); + + if (fName == null || fName.trim().length() == 0){ + fName = metadata.get(Metadata.TIKA_MIME_FILE); + } + if (fName == null){ + String ext = ".unk"; + try { + MimeType mime = TikaConfig.getDefaultConfig().getMimeRepository().forName(metadata.get(Metadata.CONTENT_TYPE)); + ext = mime.getExtension(); + } catch (MimeTypeException e1) { + + } + fName = "file"+i++ +ext; + } + Metadata targ = new Metadata(); + for (String name : metadata.names()){ + String[] values = metadata.getValues(name); + for (int i = 0; i < values.length; i++){ + targ.add(name, values[i]); + } + } + metadatas.put(fName, targ); + } + + public Map getAllMetadata(){ + return metadatas; + } + } } Index: tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java (revision 1577107) +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java (working copy) @@ -205,6 +205,7 @@ private final XHTMLContentHandler out; private final Metadata metadata; + private final RTFEmbObjHandler embObjHandler; // Used when extracting CREATION date: private int year, month, day, hour, minute; @@ -327,9 +328,11 @@ ANSICPG_MAP.put(57011, WINDOWS_57011); // Punjabi } - public TextExtractor(XHTMLContentHandler out, Metadata metadata) { + public TextExtractor(XHTMLContentHandler out, Metadata metadata, + RTFEmbObjHandler embObjHandler) { this.metadata = metadata; this.out = out; + this.embObjHandler = embObjHandler; } public boolean isIgnoringLists() { @@ -340,7 +343,7 @@ this.ignoreLists = ignore; } - private static boolean isHexChar(int ch) { + protected static boolean isHexChar(int ch) { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); @@ -355,7 +358,7 @@ return ch >= '0' && ch <= '9'; } - private static int hexValue(int ch) { + protected static int hexValue(int ch) { if (ch >= '0' && ch <= '9') { return ch - '0'; } else if (ch >= 'a' && ch <= 'z') { @@ -417,6 +420,8 @@ if (inHeader || fieldState == 1) { pendingBuffer.append(ch); + } else if (groupState.sn == true || groupState.sv == true) { + embObjHandler.writeMetadataChar(ch); } else { if (pendingCharCount == pendingChars.length) { // Gradual but exponential growth: @@ -467,7 +472,12 @@ // parsed document closing brace break; } - } else if (b != '\r' && b != '\n' && (!groupState.ignore || nextMetaData != null)) { + } else if (groupState.objdata == true || + groupState.pictDepth == 1) { + embObjHandler.writeHexChar(b); + } else if(b != '\r' && b != '\n' && (!groupState.ignore || + nextMetaData != null || groupState.sn == true || + groupState.sv == true)) { // Linefeed and carriage return are not // significant if (ansiSkip != 0) { @@ -924,7 +934,7 @@ // in the header can be unicode escaped as well: if (equals("u")) { // Unicode escape - if (!groupState.ignore) { + if (!groupState.ignore || groupState.sv || groupState.sn) { final char utf16CodeUnit = (char) (param & 0xffff); addOutputChar(utf16CodeUnit); } @@ -938,14 +948,25 @@ groupState.ucSkip = (int) param; } else if (equals("bin")) { if (param >= 0) { - int bytesToRead = param; - byte[] tmpArray = new byte[Math.min(1024, bytesToRead)]; - while (bytesToRead > 0) { - int r = in.read(tmpArray, 0, Math.min(bytesToRead, tmpArray.length)); - if (r < 0) { - throw new TikaException("unexpected end of file: need " + param + " bytes of binary data, found " + (param-bytesToRead)); + if (groupState.pictDepth == 1) { + try { + embObjHandler.writeBytes(in, param); + } catch (IOException e) { + //param was out of bounds or something went wrong + //during writing; skip this object and move on. + embObjHandler.reset(); } - bytesToRead -= r; + } else { + int bytesToRead = param; + byte[] tmpArray = new byte[Math.min(1024, bytesToRead)]; + while (bytesToRead > 0) { + int r = in.read(tmpArray, 0, Math.min(bytesToRead, tmpArray.length)); + if (r < 0) { + throw new TikaException("unexpected end of file: need " + + param + " bytes of binary data, found " + (param-bytesToRead)); + } + bytesToRead -= r; + } } } else { // log some warning? @@ -1156,11 +1177,25 @@ // TODO: we should produce a table output here? //addOutputChar(' '); endParagraph(true); + } else if (equals("sp")) { + groupState.sp = true; + } else if (equals("sn")) { + embObjHandler.startSN(); + groupState.sn = true; + } else if (equals("sv")) { + embObjHandler.startSV(); + groupState.sv = true; + } else if (equals("object")) { + pushText(); + embObjHandler.setInObject(true); + groupState.object = true; + } else if (equals("objdata")) { + groupState.objdata = true; + embObjHandler.startObjData(); } else if (equals("pict")) { pushText(); - // TODO: create img tag? but can that support - // embedded image data? - groupState.ignore = true; + groupState.pictDepth = 1; + embObjHandler.startPict(); } else if (equals("line")) { if (!ignored) { addOutputChar('\n'); @@ -1310,6 +1345,25 @@ assert groupState.depth > 0; ansiSkip = 0; + //embedded obj/pict state + if (groupState.objdata == true){ + embObjHandler.handleCompletedObject(); + groupState.objdata = false; + } else if (groupState.pictDepth > 0){ + if (groupState.sn == true) { + embObjHandler.endSN(); + } else if (groupState.sv == true) { + embObjHandler.endSV(); + } else if (groupState.sp == true) { + embObjHandler.endSP(); + } else if (groupState.pictDepth == 1) { + embObjHandler.handleCompletedObject(); + } + } + if (groupState.object) { + embObjHandler.setInObject(false); + } + // Be robust if RTF doc is corrupt (has too many // closing }s): // TODO: log a warning? Index: tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFParser.java (revision 1577107) +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFParser.java (working copy) @@ -46,13 +46,43 @@ return SUPPORTED_TYPES; } + /** maximum number of bytes per embedded object/pict (default: 20MB)*/ + private static int EMB_OBJ_MAX_BYTES = 20*1028*1028; //20MB + + /** + * Bytes for embedded objects are currently cached in memory. + * If something goes wrong during the parsing of an embedded object, + * it is possible that a read length may be crazily too long + * and cause a heap crash. + * + * @param max maximum number of bytes to allow for embedded objects. If + * the embedded object has more than this number of bytes, skip it. + */ + public static void setMaxBytesForEmbeddedObject(int max) { + EMB_OBJ_MAX_BYTES = max; + } + + /** + * See {@link #setMaxBytesForEmbeddedObject(int)}. + * + * @return maximum number of bytes allowed for an embedded object. + * + */ + public static int getMaxBytesForEmbeddedObject() { + return EMB_OBJ_MAX_BYTES; + } + public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { TaggedInputStream tagged = new TaggedInputStream(stream); try { - final TextExtractor ert = new TextExtractor(new XHTMLContentHandler(handler, metadata), metadata); + RTFEmbObjHandler embObjHandler = new RTFEmbObjHandler(handler, + metadata, context); + final TextExtractor ert = + new TextExtractor(new XHTMLContentHandler(handler, + metadata), metadata, embObjHandler); ert.extract(stream); metadata.add(Metadata.CONTENT_TYPE, "application/rtf"); } catch (IOException e) { Index: tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFEmbObjHandler.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFEmbObjHandler.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFEmbObjHandler.java (revision 0) @@ -0,0 +1,240 @@ +package org.apache.tika.parser.rtf; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor; +import org.apache.tika.io.FilenameUtils; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.RTFMetadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.EmbeddedContentHandler; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +/** + * This class buffers data from embedded objects and pictures. + * + * When the parser has finished an object or picture and called + * {@link #handleCompletedObject()}, this will write the object + * to the {@link #handler}. + * + * This (in combination with TextExtractor) expects basically a flat parse. It will pull out + * all pict whether they are tied to objdata or are intended + * to be standalone. + * + * This tries to pull metadata around a pict that is encoded + * with {sp {sn} {sv}} types of data. This information + * sometimes contains the name and even full file path of the original file. + * + * + * + */ +class RTFEmbObjHandler { + + private static final String EMPTY_STRING = ""; + + private enum EMB_STATE { + PICT, //recording pict data + OBJDATA, //recording objdata + NADA + }; + + //high hex cached for writing hexpair chars (data) + private int hi = -1; + + private boolean inObject = false; + + private String sv = EMPTY_STRING; + private String sn = EMPTY_STRING; + + private StringBuilder sb = new StringBuilder(); + + private final ContentHandler handler; + private Metadata metadata; + private final ParseContext context; + + private final ByteArrayOutputStream os; + private EMB_STATE state = EMB_STATE.NADA; + + protected RTFEmbObjHandler(ContentHandler handler,Metadata metadata, ParseContext context) { + this.handler = handler; + + this.context = context; + os = new ByteArrayOutputStream(); + } + + protected void startPict() { + state = EMB_STATE.PICT; + metadata = new Metadata(); + } + + protected void startObjData() { + state = EMB_STATE.OBJDATA; + metadata = new Metadata(); + } + + protected void startSN() { + sb.setLength(0); + sb.append(RTFMetadata.RTF_PICT_META_PREFIX); + } + + protected void endSN() { + sn = sb.toString(); + } + + protected void startSV() { + sb.setLength(0); + } + + protected void endSV() { + sv = sb.toString(); + } + + //end metadata pair + protected void endSP() { + metadata.add(sn, sv); + } + + protected void setInObject(boolean v) { + inObject = v; + } + + protected boolean getInObject() { + return inObject; + } + + protected void writeMetadataChar(char c) { + sb.append(c); + } + + protected void writeHexChar(int b) throws IOException, TikaException { + //if not hexchar, ignore + //white space is common + if (TextExtractor.isHexChar(b)) { + if (hi == -1) { + hi = 16*TextExtractor.hexValue(b); + } else { + long sum = hi+TextExtractor.hexValue(b); + if (sum > Integer.MAX_VALUE || sum < 0) { + throw new IOException("hex char to byte overflow"); + } + os.write((int)sum); + hi = -1; + } + return; + } + if (b == -1) { + throw new TikaException("hit end of stream before finishing byte pair"); + } + } + + + protected void writeBytes(InputStream is, int len) throws IOException, TikaException { + if (len < 0 || len > RTFParser.getMaxBytesForEmbeddedObject()) { + throw new IOException("length of bytes to read out of bounds: " + len); + } + + byte[] bytes = new byte[len]; + int bytesRead = is.read(bytes); + if (bytesRead < len) { + throw new TikaException("unexpected end of file: need " + len + + " bytes of binary data, found " + (len-bytesRead)); + } + os.write(bytes); + } + + /** + * Call this when the objdata/pict has completed + * @throws IOException + * @throws SAXException + * @throws TikaException + */ + protected void handleCompletedObject() throws IOException, SAXException, TikaException { + EmbeddedDocumentExtractor embeddedExtractor = context.get(EmbeddedDocumentExtractor.class); + + if (embeddedExtractor == null) { + embeddedExtractor = new ParsingEmbeddedDocumentExtractor(context); + } + + byte[] bytes = os.toByteArray(); + + if (state == EMB_STATE.OBJDATA) { + RTFObjDataParser objParser = new RTFObjDataParser(); + try{ + byte[] objBytes = objParser.parse(bytes, metadata); + + extractObj(objBytes, handler, embeddedExtractor, metadata); + } catch (IOException e) { + e.printStackTrace(); + } + } else if (state == EMB_STATE.PICT) { + String filePath = metadata.get(RTFMetadata.RTF_PICT_META_PREFIX+"wzDescription"); + if (filePath != null && filePath.length() > 0){ + metadata.set(Metadata.EMBEDDED_RELATIONSHIP_ID, filePath); + metadata.set(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getSafeFilename(filePath)); + } + extractObj(bytes, handler, embeddedExtractor, metadata); + metadata.set(RTFMetadata.PICT_IN_OBJECT, Boolean.toString(inObject)); + + } else if (state == EMB_STATE.NADA) { + //swallow...no start for pict or embed?! + } + reset(); + } + + protected void reset() { + state = EMB_STATE.NADA; + os.reset(); + metadata = new Metadata(); + hi = -1; + sv = EMPTY_STRING; + sn = EMPTY_STRING; + sb.setLength(0); + inObject = false; + } + + private void extractObj(byte[] bytes, ContentHandler handler, + EmbeddedDocumentExtractor embeddedExtractor, Metadata metadata) + throws SAXException, IOException, TikaException { + + if (bytes == null) { + return; + } + + metadata.set(Metadata.CONTENT_LENGTH, Long.toString(bytes.length)); + + if (embeddedExtractor.shouldParseEmbedded(metadata)) { + TikaInputStream stream = TikaInputStream.get(bytes); + try { + embeddedExtractor.parseEmbedded( + stream, + new EmbeddedContentHandler(handler), + metadata, false); + } finally { + stream.close(); + } + } + } +} Index: tika-parsers/src/main/java/org/apache/tika/parser/rtf/GroupState.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/GroupState.java (revision 1577107) +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/GroupState.java (working copy) @@ -33,6 +33,18 @@ public int list; public int listLevel; public Charset fontCharset; + //in objdata + public boolean objdata; + //depth in pict, 1 = at pict level + public int pictDepth; + //in picprop key/value pair + public boolean sp; + //in picprop's name + public boolean sn; + //in picprop's value + public boolean sv; + //in embedded object or not + public boolean object; // Create default (root) GroupState public GroupState() { @@ -48,5 +60,8 @@ listLevel = other.listLevel; fontCharset = other.fontCharset; depth = 1+other.depth; + pictDepth = other.pictDepth > 0 ? other.pictDepth + 1 : 0; + //do not inherit object, sn, sv or sp + } } Index: tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFObjDataParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFObjDataParser.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/RTFObjDataParser.java (revision 0) @@ -0,0 +1,317 @@ +package org.apache.tika.parser.rtf; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +import org.apache.poi.poifs.filesystem.DirectoryNode; +import org.apache.poi.poifs.filesystem.DocumentEntry; +import org.apache.poi.poifs.filesystem.DocumentInputStream; +import org.apache.poi.poifs.filesystem.Entry; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; +import org.apache.poi.poifs.filesystem.Ole10Native; +import org.apache.poi.poifs.filesystem.Ole10NativeException; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.util.IOUtils; +import org.apache.tika.io.FilenameUtils; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.RTFMetadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.microsoft.OfficeParser.POIFSDocumentType; +import org.apache.tika.parser.pkg.ZipContainerDetector; + +/** + * Many thanks to Simon Mourier for: + * http://stackoverflow.com/questions/14779647/extract-embedded-image-object-in-rtf + * and for granting permission to use his code in Tika. + * + */ +class RTFObjDataParser { + + private final static int[] int32Pows = new int[]{ + 1, 256, 65536, 16777216 + }; + + private final static String WIN_ASCII = "WINDOWS-1252"; + + /** + * Parses the embedded object/pict string + * + * @param bytes actual bytes (already converted from the + * hex pair string stored in the embedded object data into actual bytes or read + * as raw binary bytes) + * @return a SimpleRTFEmbObj or null + * @throws IOException if there are any surprises during parsing + */ + protected byte[] parse(byte[] bytes, Metadata metadata) + throws IOException { + ByteArrayInputStream is = new ByteArrayInputStream(bytes); + int version = readInt32(is); //version (do anything with this? + metadata.add("RTF_EMB_OBJ_APP_VERSION", Integer.toString(version)); + + int formatId = readInt32(is); + //2 is an embedded object. 1 is a link. + if (formatId != 2) { + return null; + } + String className = readLengthPrefixedAnsiString(is).trim(); + String topicName = readLengthPrefixedAnsiString(is).trim(); + String itemName = readLengthPrefixedAnsiString(is).trim(); + + if (className != null && className.length() > 0) { + metadata.add(RTFMetadata.EMB_CLASS, className); + } + if (topicName != null && topicName.length() > 0) { + metadata.add(RTFMetadata.EMB_TOPIC, topicName); + } + if (itemName != null && itemName.length() > 0) { + metadata.add(RTFMetadata.EMB_ITEM, itemName); + } + + int dataSz = readInt32(is); + + byte[] embObjBytes = readBytes(is, dataSz); + + if (className.toLowerCase().equals("package")){ + return handlePackage(embObjBytes, metadata); + //else if (className.toLowerCase().equals("pbrush")) { + //simple bitmap bytes + // return embObjBytes; + } else { + ByteArrayInputStream embIs = new ByteArrayInputStream(embObjBytes); + if (NPOIFSFileSystem.hasPOIFSHeader(embIs)){ + byte[] ret = null; + try{ + ret = handleEmbeddedPOIFS(embIs, metadata); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (embIs != null) { + embIs.close(); + } + } + return ret; + } + if (embIs != null) { + embIs.close(); + } + } + return embObjBytes; + } + + + //will throw IOException if not actually POIFS + //can return null byte[] + private byte[] handleEmbeddedPOIFS(InputStream is, Metadata metadata) + throws IOException { + + NPOIFSFileSystem fs = null; + byte[] ret = null; + try { + + fs = new NPOIFSFileSystem(is);//new ByteArrayInputStream(bytes)); + + DirectoryNode root = fs.getRoot(); + + if (root == null) { + return ret; + } + if (root.hasEntry("Package")){ + Entry ooxml = root.getEntry("Package"); + TikaInputStream stream = TikaInputStream.get(new DocumentInputStream((DocumentEntry) ooxml)); + // ZipContainerDetector zd = new ZipContainerDetector(); + // MediaType type = zd.detect(stream, new Metadata()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + IOUtils.copy(stream, out); + // metadata.add(Metadata.CONTENT_TYPE, type.getSubtype()); + + ret = out.toByteArray(); + } else { + POIFSDocumentType type = POIFSDocumentType.detectType(root); + + if (type == POIFSDocumentType.OLE10_NATIVE) { + try { + // Try to un-wrap the OLE10Native record: + Ole10Native ole = Ole10Native.createFromEmbeddedOleObject((DirectoryNode)root); + ret = ole.getDataBuffer(); + } catch (Ole10NativeException ex) { + // Not a valid OLE10Native record, skip it + } + } else if (type == POIFSDocumentType.COMP_OBJ) { + + DocumentEntry contentsEntry; + try { + contentsEntry = (DocumentEntry)root.getEntry("CONTENTS"); + } catch (FileNotFoundException ioe) { + contentsEntry = (DocumentEntry)root.getEntry("Contents"); + } + + DocumentInputStream inp = null; + try { + inp = new DocumentInputStream(contentsEntry); + ret = new byte[contentsEntry.getSize()]; + inp.readFully(ret); + } finally { + if (inp != null) { + inp.close(); + } + } + } else { + //if (type == POIFSDocumentType.WORDDOCUMENT or anything else, return bytes + metadata.set(Metadata.CONTENT_TYPE, type.getType().getSubtype()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + is.reset(); + IOUtils.copy(is, out); + // metadata.add(Metadata.CONTENT_TYPE, type.getSubtype()); + + ret = out.toByteArray(); + } + } + } finally { + if (fs != null) { + fs.close(); + } + } + return ret; + } + + + + /** + * can return null if there is a linked object + * instead of an embedded file + */ + private byte[] handlePackage(byte[] pkgBytes, Metadata metadata) throws IOException { + //now parse the package header + ByteArrayInputStream is = new ByteArrayInputStream(pkgBytes); + readUInt16(is); + + String displayName = readAnsiString(is); + + //should we add this to the metadata? + readAnsiString(is); //iconFilePath + readUInt16(is); //iconIndex + int type = readUInt16(is); //type + + //1 is link, 3 is embedded object + //this only handles embedded objects + if (type != 3) { + return null; + } + //should we really be ignoring this filePathLen? + readInt32(is); //filePathLen + + String ansiFilePath = readAnsiString(is); //filePath + int bytesLen = readInt32(is); + + byte[] objBytes = new byte[bytesLen]; + is.read(objBytes); + + StringBuilder unicodeFilePath = new StringBuilder(); + //this might not actually exist + //have to try to readint32 to see + try { + int unicodeLen = readInt32(is); + + for (int i = 0; i < unicodeLen; i++){ + int lo = is.read(); + int hi = is.read(); + int sum = lo+256*hi; + if (hi == -1 || lo == -1){ + //stream ran out; empty SB and stop + unicodeFilePath.setLength(0); + break; + } + unicodeFilePath.append((char)sum); + } + } catch (IOException e) { + //swallow; the unicode file path is optional and might not happen + unicodeFilePath.setLength(0); + } + String fileNameToUse = ""; + String pathToUse = ""; + if (unicodeFilePath.length() > 0){ + String uPath = unicodeFilePath.toString(); + fileNameToUse = uPath; + pathToUse = uPath; + } else { + fileNameToUse = displayName == null ? "" : displayName; + pathToUse = ansiFilePath == null ? "" : ansiFilePath; + } + System.out.println("HERE!!!!!!!" + FilenameUtils.getSafeFilename(fileNameToUse)); + metadata.add(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getSafeFilename(fileNameToUse)); + metadata.add(Metadata.EMBEDDED_RELATIONSHIP_ID, pathToUse); + + return objBytes; + } + + + private int readUInt16(InputStream is) throws IOException { + int lo = is.read(); + int hi = is.read()*255; + if (lo == -1 || hi == -1) { + throw new IOException("Hit end of stream before reading int16."); + } + return hi+lo; + } + + private int readInt32(InputStream is) throws IOException { + long sum = 0; + for (int i = 0; i < 4; i++){ + int v = is.read(); + if (v == -1) { + throw new IOException("Hit end of stream before finishing Int32."); + } + sum += v*(long)int32Pows[i]; + } + if (sum < 0 || sum > Integer.MAX_VALUE) { + throw new IOException("int32 out of bounds: "+sum); + } + return (int)sum; + } + + private String readAnsiString(InputStream is) throws IOException { + StringBuilder sb = new StringBuilder(); + int c = is.read(); + while (c > 0) { + sb.append((char)c); + c = is.read(); + } + if (c == -1) { + throw new IOException("Hit end of stream before end of AnsiString"); + } + return sb.toString(); + } + + private String readLengthPrefixedAnsiString(InputStream is) throws IOException { + int len = readInt32(is); + byte[] bytes = readBytes(is, len); + try { + return new String(bytes, WIN_ASCII); + } catch (UnsupportedEncodingException e) { + //shouldn't ever happen + throw new IOException("Unsupported encoding"); + } + } + + + private byte[] readBytes(InputStream is, int len) throws IOException { + if (len < 0 || len > RTFParser.getMaxBytesForEmbeddedObject()) { + throw new IOException("Requested length for reading bytes is out of bounds: " + len); + } + byte[] bytes = new byte[len]; + int read = is.read(bytes); + if (read != len) { + throw new IOException("Hit end of stream before reading all bytes"); + } + + return bytes; + } +} Index: tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java (revision 1577107) +++ tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java (working copy) @@ -94,5 +94,28 @@ assertEquals(EXPECTED_NAME, FilenameUtils.normalize(TEST_NAME)); } + @Test + public void testSafeFileName() throws Exception { + String s = "C:\\the\\quick.ppt"; + assertEquals("quick.ppt", FilenameUtils.getSafeFilename(s)); + + s = "/the/quick.ppt"; + assertEquals("quick.ppt", FilenameUtils.getSafeFilename(s)); + + s = "/the/quick/"; + assertEquals("quick", FilenameUtils.getSafeFilename(s)); + s = "~/the/quick////\\\\//"; + assertEquals("quick", FilenameUtils.getSafeFilename(s)); + s = "////"; + assertEquals("", FilenameUtils.getSafeFilename(s)); + + s = "C:////"; + assertEquals("", FilenameUtils.getSafeFilename(s)); + + s = "C:////.."; + assertEquals("", FilenameUtils.getSafeFilename(s)); + + } + } Index: tika-core/src/main/java/org/apache/tika/metadata/RTFMetadata.java =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/RTFMetadata.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/metadata/RTFMetadata.java (revision 0) @@ -0,0 +1,51 @@ +package org.apache.tika.metadata; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; + +public interface RTFMetadata { + + public static final String PREFIX_RTF_META = "rtf_meta"; + + + public static final String RTF_PICT_META_PREFIX = "rtf_pict:"; + + /** + * if set to true, this means that an image file is probably a "thumbnail" + */ + Property PICT_IN_OBJECT = Property.internalBoolean(PREFIX_RTF_META+ + Metadata.NAMESPACE_PREFIX_DELIMITER+"pict_in_object"); + + /** + * if an application and version is given as part of the + * embedded object, this is the literal string + */ + Property EMB_APP_VERSION = Property.internalText(PREFIX_RTF_META+ + Metadata.NAMESPACE_PREFIX_DELIMITER+"emb_app_version"); + + Property EMB_CLASS = Property.internalText(PREFIX_RTF_META+ + Metadata.NAMESPACE_PREFIX_DELIMITER+"emb_class"); + + Property EMB_TOPIC = Property.internalText(PREFIX_RTF_META+ + Metadata.NAMESPACE_PREFIX_DELIMITER+"emb_topic"); + + Property EMB_ITEM = Property.internalText(PREFIX_RTF_META+ + Metadata.NAMESPACE_PREFIX_DELIMITER+"emb_item"); + +} Index: tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java =================================================================== --- tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java (revision 1577107) +++ tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java (working copy) @@ -17,6 +17,8 @@ package org.apache.tika.io; import java.util.HashSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class FilenameUtils { @@ -70,4 +72,59 @@ return sb.toString(); } + + /** + * This is a stand-in for a small amount of functionality + * available in commons io FilenameUtils. If Java's File were + * able handle Windows file paths correctly in linux, + * we wouldn't need this. + *

+ * The goal of this is to get a filename from a path. + * The package parsers and some other embedded doc + * extractors could put anything into Metadata.RESOURCE_NAME_KEY. + *

+ * If a careless client used that filename as if it were a + * filename and not a path when writing embedded files, + * bad things could happen. Consider: "../../../my_ppt.ppt". + *

+ * Consider using this in combination with {@link #normalize(String)}. + * + * @param path path to strip + * @return empty string or a filename + */ + public static String getSafeFilename(String path) { + + if (path == null || path.length() == 0) { + return ""; + } + + //starting at the end find the first non-slash + int end = path.length(); + for (int i = path.length()-1; i >= 0; i--) { + if (path.charAt(i) == '\\' || path.charAt(i) == '/'){ + //keep going; + } else { + break; + } + end = i; + } + //strip off initial drive info or initial slashes + String cleanPath = path.substring(0, end); + Matcher m = Pattern.compile("(?i)^(?:[a-z]:[//\\\\]*|~*(?:[//\\\\]*))").matcher(cleanPath); + cleanPath = m.replaceAll(""); + + //now find last slash + int bwdEnd = cleanPath.lastIndexOf("\\"); + int fwdEnd = cleanPath.lastIndexOf("/"); + end = Math.max(bwdEnd, fwdEnd); + if (end > -1) { + cleanPath = cleanPath.substring(end+1); + } + + //strip relative and present directory + if (cleanPath.equals("..") || cleanPath.equals(".")){ + return ""; + } + return cleanPath; + } }