Index: src/test/java/org/apache/tika/TestParsers.java =================================================================== --- src/test/java/org/apache/tika/TestParsers.java (revision 584800) +++ src/test/java/org/apache/tika/TestParsers.java (working copy) @@ -31,6 +31,7 @@ import org.apache.tika.parser.ParserFactory; import org.apache.tika.utils.ParseUtils; import org.apache.tika.utils.Utils; +import org.apache.tika.exception.TikaException; import org.jdom.JDOMException; import org.xml.sax.helpers.DefaultHandler; @@ -43,7 +44,7 @@ private File testFilesBaseDir; - public void setUp() throws JDOMException, IOException { + public void setUp() throws TikaException { /* * FIXME the old mechanism does not work anymore when running the tests * with Maven - need a resource-based one, but this means more changes @@ -68,41 +69,65 @@ public void testPDFExtraction() throws Exception { File file = getTestFile("testPDF.pdf"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, "application/pdf"); - String s3 = ParseUtils.getStringContent(file, TikaConfig - .getDefaultConfig()); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser("application/pdf", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); - assertEquals(s1, s3); } public void testTXTExtraction() throws Exception { File file = getTestFile("testTXT.txt"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, "text/plain"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser("text/plain", null); + String s2 = ParseUtils.parse(file, null, parser2); + + assertEquals(s1, s2); } public void testRTFExtraction() throws Exception { File file = getTestFile("testRTF.rtf"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, "application/rtf"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser("application/rtf", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); } public void testXMLExtraction() throws Exception { File file = getTestFile("testXML.xml"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, "application/xml"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser("application/xml", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); } public void testPPTExtraction() throws Exception { File file = getTestFile("testPPT.ppt"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent( - file, tc, "application/vnd.ms-powerpoint"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = + ParseUtils.getParser("application/vnd.ms-powerpoint", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); + ParserConfig config = tc.getParserConfig("application/vnd.ms-powerpoint"); Parser parser = ParserFactory.getParser(config); @@ -118,9 +143,15 @@ public void testWORDxtraction() throws Exception { File file = getTestFile("testWORD.doc"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, "application/msword"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser("application/msword", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); + ParserConfig config = tc.getParserConfig("application/msword"); Parser parser = ParserFactory.getParser(config); Metadata metadata = new Metadata(); @@ -140,10 +171,16 @@ + "14.0 196.0 15.0 225.0 Written and saved in Microsoft Excel " + "X for Mac Service Release 1."; File file = getTestFile("testEXCEL.xls"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, - "application/vnd.ms-excel"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = + ParseUtils.getParser("application/vnd.ms-excel", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); + assertTrue("Text does not contain '" + expected + "'", s1 .contains(expected)); ParserConfig config = tc.getParserConfig("application/vnd.ms-excel"); @@ -160,16 +197,26 @@ public void testOOExtraction() throws Exception { File file = getTestFile("testOpenOffice2.odt"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, - "application/vnd.oasis.opendocument.text"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser( + "application/vnd.oasis.opendocument.text", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); } public void testHTMLExtraction() throws Exception { File file = getTestFile("testHTML.html"); - String s1 = ParseUtils.getStringContent(file, tc); - String s2 = ParseUtils.getStringContent(file, tc, "text/html"); + + Parser parser1 = ParseUtils.getParser(file, tc); + String s1 = ParseUtils.parse(file, null, parser1); + + Parser parser2 = ParseUtils.getParser("text/html", null); + String s2 = ParseUtils.parse(file, null, parser2); + assertEquals(s1, s2); ParserConfig config = tc.getParserConfig("text/html"); @@ -215,8 +262,11 @@ } } + public void testDefaultConfig() throws TikaException { + assertNotNull(TikaConfig.getDefaultConfig()); + } + private File getTestFile(String filename) { return new File(testFilesBaseDir, filename); } - } Index: src/main/java/org/apache/tika/utils/ParseUtils.java =================================================================== --- src/main/java/org/apache/tika/utils/ParseUtils.java (revision 584800) +++ src/main/java/org/apache/tika/utils/ParseUtils.java (working copy) @@ -17,7 +17,6 @@ package org.apache.tika.utils; //JDK imports -import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -28,6 +27,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.tika.config.ParserConfig; import org.apache.tika.config.TikaConfig; import org.apache.tika.exception.TikaException; @@ -45,65 +45,80 @@ public class ParseUtils implements TikaMimeKeys { /** - * Returns a parser that can handle the specified MIME type, and is set to - * receive input from a stream opened from the specified URL. NB: Close the - * input stream when it is no longer needed! + * Returns a parser that can handle the specified MIME type. * - * @param config + * @param tikaConfig * @param mimeType * the document's MIME type * @return a parser appropriate to this MIME type * @throws TikaException */ - public static Parser getParser(String mimeType, TikaConfig config) + public static Parser getParser(String mimeType, TikaConfig tikaConfig) throws TikaException { - ParserConfig pc = config.getParserConfig(mimeType); + + ensureNotNull(mimeType, "MIME type"); + + if (tikaConfig == null) { + tikaConfig = TikaConfig.getDefaultConfig(); + } + ParserConfig pc = tikaConfig.getParserConfig(mimeType); return ParserFactory.getParser(pc); } /** - * Returns a parser that can handle the specified MIME type, and is set to - * receive input from a stream opened from the specified URL. The MIME type - * is determined automatically. NB: Close the input stream when it is no - * longer needed! + * Returns a parser that can handle the MIME type of the document, which + * is determined automatically. * * @param documentUrl * URL pointing to the document to parse - * @param config + * @param tikaConfig may be null for default configuration * @return a parser appropriate to this MIME type and ready to read input * from the specified document * @throws TikaException */ - public static Parser getParser(URL documentUrl, TikaConfig config) + public static Parser getParser(URL documentUrl, TikaConfig tikaConfig) throws TikaException { - String mimetype = config.getMimeRepository().getMimeType(documentUrl) - .getName(); - return getParser(mimetype, config); + + ensureNotNull(documentUrl, "Document URL"); + + if (tikaConfig == null) { + tikaConfig = TikaConfig.getDefaultConfig(); + } + + String mimetype = tikaConfig.getMimeRepository() + .getMimeType(documentUrl).getName(); + return getParser(mimetype, tikaConfig); } /** - * Returns a parser that can handle the specified MIME type, and is set to - * receive input from a stream opened from the specified URL. NB: Close the - * input stream when it is no longer needed! - * + * Returns a parser that can handle the MIME type of the document, which + * is determined automatically. + * * @param documentFile * File object pointing to the document to parse - * @param config + * @param tikaConfig * @return a parser appropriate to this MIME type and ready to read input * from the specified document * @throws TikaException */ - public static Parser getParser(File documentFile, TikaConfig config) + public static Parser getParser(File documentFile, TikaConfig tikaConfig) throws TikaException { - String mimetype = config.getMimeRepository().getMimeType(documentFile) - .getName(); - return getParser(mimetype, config); + + ensureNotNull(documentFile, "Document File"); + + if (tikaConfig == null) { + tikaConfig = TikaConfig.getDefaultConfig(); + } + + String mimetype = tikaConfig.getMimeRepository() + .getMimeType(documentFile).getName(); + return getParser(mimetype, tikaConfig); } /** * Returns a list of parsers from zip InputStream * - * @param zip + * @param zipIs * InputStream * @param config * @return a list of parsers from zip file @@ -133,7 +148,7 @@ public static List getParsersFromZip(File zip, TikaConfig config) throws TikaException, FileNotFoundException { String zipMimeType = config.getMimeRepository().getMimeType(zip) - .getName(); + .getName(); if (!zipMimeType.equalsIgnoreCase("application/zip")) { throw new TikaException("The file you are using is note a zip file"); } @@ -143,7 +158,7 @@ /** * Returns a list of parsers from URL * - * @param URL + * @param zip * @param config * @return a list of parsers from zip file * @throws TikaException @@ -152,7 +167,7 @@ public static List getParsersFromZip(URL zip, TikaConfig config) throws TikaException, IOException { String zipMimeType = config.getMimeRepository().getMimeType(zip) - .getName(); + .getName(); if (!zipMimeType.equalsIgnoreCase("application/zip")) { throw new TikaException("The file you are using is note a zip file"); } @@ -160,99 +175,195 @@ } /** - * Gets the string content of a document read from an input stream. - * - * @param stream the stream from which to read document data - * @param config - * @param mimeType MIME type of the data + * Gets the string content and optionally metadata of a document + * read from an input stream. + * + * @param documentFile File pointing to the document to parse + * @param metadata instance in which to put metadata, may be null + * @param parser parser to use to parser the input stream * @return the string content parsed from the document */ - public static String getStringContent( - InputStream stream, TikaConfig config, String mimeType) - throws TikaException, IOException { + public static String parse(File documentFile, Metadata metadata, + Parser parser) + throws TikaException { + + ensureNotNull(documentFile, "Document File"); + ensureNotNull(parser, "Parser"); + + InputStream inputStream = null; + try { - ParserConfig pc = config.getParserConfig(mimeType); - Parser parser = ParserFactory.getParser(pc); - StringWriter writer = new StringWriter(); - parser.parse( - stream, new WriteOutContentHandler(writer), new Metadata()); - return writer.toString(); - } catch (SAXException e) { - throw new TikaException("Unexpected SAX error", e); + inputStream = new FileInputStream(documentFile); + return parseAndClose(inputStream, metadata, parser); + } catch (IOException e) { + throw new TikaException( + "I/O error during parse or opening of stream: " + + documentFile + , e); } } + /** - * Gets the string content of a document read from an input stream. - * + * Gets the string content and optionally metadata of a document + * read from an input stream. + * * @param documentUrl * URL pointing to the document to parse - * @param config + * @param metadata instance in which to put metadata, may be null + * @param parser parser to use to parser the input stream * @return the string content parsed from the document */ - public static String getStringContent(URL documentUrl, TikaConfig config) - throws TikaException, IOException { - String mime = config.getMimeRepository().getMimeType(documentUrl) - .getName(); - return getStringContent(documentUrl, config, mime); + public static String parse(URL documentUrl, Metadata metadata, + Parser parser) + throws TikaException { + + ensureNotNull(documentUrl, "Document URL"); + ensureNotNull(parser, "Parser"); + + InputStream inputStream = null; + + try { + inputStream = documentUrl.openStream(); + return parseAndClose(inputStream, metadata, parser); + } catch (IOException e) { + throw new TikaException( + "I/O error during opening or parse of stream: " + + documentUrl + , e); + } } /** - * Gets the string content of a document read from an input stream. - * - * @param documentUrl - * URL pointing to the document to parse - * @param config - * @param mimeType - * MIME type of the data - * @return the string content parsed from the document + * Parses an InputStream. If a Metadata instance is passed, it will be + * used by the parsers to populate the metadata. If null is passed instead, + * a new instance will be created (but not available to the caller). + * + * @param inputStream the stream to parse + * @param metadata instance in which to put metadata, may be null + * @param parser parser to use to parser the input stream + * @return the fulltext of the document + * @throws TikaException */ - public static String getStringContent( - URL documentUrl, TikaConfig config, String mimeType) - throws TikaException, IOException { - InputStream stream = documentUrl.openStream(); + public static String parse(InputStream inputStream, Metadata metadata, + Parser parser) throws TikaException { + + ensureNotNull(inputStream, "Input Stream"); + ensureNotNull(parser, "Parser"); + + StringWriter fullTextStringWriter = new StringWriter(); + WriteOutContentHandler handler + = new WriteOutContentHandler(fullTextStringWriter); + + // So the parsers don't need to check for null... + if (metadata == null) { + metadata = new Metadata(); + } + + String fulltext = null; try { - return getStringContent(stream, config, mimeType); - } finally { - stream.close(); + parser.parse(inputStream, handler, metadata); + fulltext = fullTextStringWriter.toString(); + } catch (IOException e) { + String message = buildExceptionMessage( + "I/O error occurred during parse.", metadata); + throw new TikaException(message, e); + } catch (SAXException e) { + String message = buildExceptionMessage( + "SAX error occurred during parse.", metadata); + throw new TikaException(message, e); } + + // Note: There is no need to close() a StringWriter. + + return fulltext; } + + /** - * Gets the string content of a document read from an input stream. - * - * @param documentFile - * File object pointing to the document to parse - * @param config - * @param mimeType - * MIME type of the data - * @return the string content parsed from the document + * Parses an InputStream and then closes it. + * If a Metadata instance is passed, it will be + * used by the parsers to populate the metadata. If null is passed instead, + * a new instance will be created (but not available to the caller). + * + * This method is called by parse methods that take resource identifiers + * as parameters and need to guarantee that the streams opened from + * those resources are closed. + * + * @param inputStream the stream to parse + * @param metadata instance in which to put metadata, may be null + * @param parser parser to use to parser the input stream + * @return the fulltext of the document + * @throws TikaException */ - public static String getStringContent( - File documentFile, TikaConfig config, String mimeType) - throws TikaException, IOException { - InputStream stream = new BufferedInputStream(new FileInputStream( - documentFile)); + public static String parseAndClose(InputStream inputStream, + Metadata metadata, Parser parser) + throws TikaException { + + ensureNotNull(inputStream, "Input Stream"); + ensureNotNull(parser, "Parser"); + try { - return getStringContent(stream, config, mimeType); + return parse(inputStream, metadata, parser); } finally { - stream.close(); + if (inputStream != null){ + try { + inputStream.close(); + } catch (IOException e) { + // Do nothing; would rather use IOUtils.closeQuietly() + // here if we can include Commons IO as a dependency. + } + } } } /** - * Gets the string content of a document read from an input stream. - * - * @param documentFile - * File object pointing to the document to parse - * @param config - * @return the string content parsed from the document + * Returns the resource name (String, File, URL, etc.) + * associated with this stream. + * + * @param metadata metadata instance to query for the resource name + * @return the resource name */ - public static String getStringContent(File documentFile, TikaConfig config) - throws TikaException, IOException { - String mime = - config.getMimeRepository().getMimeType(documentFile).getName(); - return getStringContent(documentFile, config, mime); + public static String getResourceName(Metadata metadata) { + + String resourceName = null; + + // TODO: This may well not be the best place to put this method, + // nor the best name for the property. + + if (metadata != null) { + resourceName = metadata.get("resourceName"); + } + + return resourceName; } + + /** + * If resource name is not empty, adds ": " + the resource name to + * the root message. + */ + private static String buildExceptionMessage( + String root, Metadata metadata) { + + String resourceName = getResourceName(metadata); + + return StringUtils.isNotBlank(resourceName) + ? root + ": " + resourceName + : root; + } + + + private static void ensureNotNull(Object object, Object identifier) + throws TikaException { + + if (object == null) { + String s = identifier != null + ? identifier.toString() + : "[no identifier]"; + throw new TikaException(s + " value was null."); + } + + } } Index: src/main/java/org/apache/tika/config/TikaConfig.java =================================================================== --- src/main/java/org/apache/tika/config/TikaConfig.java (revision 584800) +++ src/main/java/org/apache/tika/config/TikaConfig.java (working copy) @@ -25,6 +25,7 @@ import java.util.Map; //TIKA imports +import org.apache.tika.exception.TikaException; import org.apache.tika.mime.MimeTypes; import org.apache.tika.mime.MimeUtils; import org.apache.tika.utils.Utils; @@ -96,14 +97,21 @@ * return a shared instance once it is completely immutable. * * @return - * @throws IOException - * @throws JDOMException + * @throws TikaException */ - public static TikaConfig getDefaultConfig() - throws IOException, JDOMException { + public static TikaConfig getDefaultConfig() throws TikaException { - return new TikaConfig( - Utils.class.getResourceAsStream(DEFAULT_CONFIG_LOCATION)); + final String exceptionMessage = + "Exception occurred while getting default Tika configuration."; + + try { + return new TikaConfig(TikaConfig.class + .getResourceAsStream(DEFAULT_CONFIG_LOCATION)); + } catch (IOException e) { + throw new TikaException(exceptionMessage, e); + } catch (JDOMException e) { + throw new TikaException(exceptionMessage, e); + } } +} -}