Index: src/test/java/org/apache/tika/TestParsers.java =================================================================== --- src/test/java/org/apache/tika/TestParsers.java (revision 582120) +++ src/test/java/org/apache/tika/TestParsers.java (working copy) @@ -121,7 +121,7 @@ assertNotNull(contents); InputStream stream = new FileInputStream(file); try { - parser.getContents(stream, contents); + parser.parse(stream, contents.values()); } finally { stream.close(); } @@ -140,7 +140,7 @@ assertNotNull(contents); InputStream stream = new FileInputStream(file); try { - parser.getContents(stream, contents); + parser.parse(stream, contents.values()); } finally { stream.close(); } @@ -166,7 +166,7 @@ assertNotNull(contents); InputStream stream = new FileInputStream(file); try { - parser.getContents(stream, contents); + parser.parse(stream, contents.values()); } finally { stream.close(); } @@ -190,23 +190,18 @@ ParserConfig config = tc.getParserConfig("text/html"); Parser parser = ParserFactory.getParser(config); assertNotNull(parser); - assertEquals("org.apache.tika.parser.html.HtmlParser", parser - .getClass().getName()); - parser.setMimeType("text/html"); Map contents = config.getContents(); assertNotNull(contents); InputStream stream = new FileInputStream(file); try { - parser.getContents(stream, contents); + parser.parse(stream, contents.values()); } finally { stream.close(); } assertEquals("Title : Test Indexation Html", contents.get("title") .getValue()); - assertEquals("text/html", parser.getMimeType()); - final String text = Utils.toString(contents); final String expected = "Test Indexation Html"; assertTrue("text contains '" + expected + "'", text.contains(expected)); @@ -219,23 +214,18 @@ for (int i = 0; i < parsers.size(); i++) { Parser zipEntryParser = parsers.get(i); assertNotNull(zipEntryParser); - assertNotNull(zipEntryParser.getMimeType()); for (int j = 0; j < zipFiles.size(); j++) { - if (zipEntryParser.getMimeType().equalsIgnoreCase( - tc.getMimeRepository().getMimeType(zipFiles.get(j)) - .getName())) { - ParserConfig config = tc.getParserConfig(zipEntryParser - .getMimeType()); - Map contents = config.getContents(); - assertNotNull(contents); - InputStream stream = new FileInputStream(zipFiles.get(j)); - try { - zipEntryParser.getContents(stream, contents); - assertNotNull(contents.get("fullText")); - } finally { - stream.close(); - } - } + ParserConfig config = tc.getParserConfig( + tc.getMimeRepository().getMimeType(zipFiles.get(j)).getName()); + Map contents = config.getContents(); + assertNotNull(contents); + InputStream stream = new FileInputStream(zipFiles.get(j)); + try { + zipEntryParser.parse(stream, contents.values()); + assertNotNull(contents.get("fullText")); + } finally { + stream.close(); + } } } Index: src/main/java/org/apache/tika/utils/ParseUtils.java =================================================================== --- src/main/java/org/apache/tika/utils/ParseUtils.java (revision 582120) +++ src/main/java/org/apache/tika/utils/ParseUtils.java (working copy) @@ -54,9 +54,7 @@ public static Parser getParser(TikaConfig config, String mimeType) throws TikaException { ParserConfig pc = config.getParserConfig(mimeType); - Parser parser = ParserFactory.getParser(pc); - parser.setMimeType(mimeType); - return parser; + return ParserFactory.getParser(pc); } /** @@ -174,8 +172,7 @@ IOException { ParserConfig pc = config.getParserConfig(mimeType); Parser parser = ParserFactory.getParser(pc); - parser.setMimeType(mimeType); - return parser.getContents(inputStream, pc.getContents()); + return parser.parse(inputStream, pc.getContents().values()); } /** Index: src/main/java/org/apache/tika/parser/txt/TXTParser.java =================================================================== --- src/main/java/org/apache/tika/parser/txt/TXTParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/txt/TXTParser.java (working copy) @@ -28,9 +28,9 @@ /** * Text parser */ -public class TXTParser extends Parser { +public class TXTParser implements Parser { - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { StringBuilder sb = new StringBuilder(); BufferedReader br = new BufferedReader(new InputStreamReader(stream)); Index: src/main/java/org/apache/tika/parser/ParserDecorator.java =================================================================== --- src/main/java/org/apache/tika/parser/ParserDecorator.java (revision 0) +++ src/main/java/org/apache/tika/parser/ParserDecorator.java (revision 0) @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.tika.config.Content; +import org.apache.tika.exception.TikaException; + +/** + * Decorator base class for the {@link Parser} interface. This class + * simply delegates all parsing calls to an underlying decorated parser + * instance. Subclasses can provide extra decoration by overriding the + * parse method. + */ +public class ParserDecorator implements Parser { + + /** + * The decorated parser instance. + */ + private final Parser parser; + + /** + * Creates a decorator for the given parser. + * + * @param parser the parser instance to be decorated + */ + public ParserDecorator(Parser parser) { + this.parser = parser; + } + + /** + * Delegates the method call to the decorated parser. Subclasses should + * override this method (and use super.parse() to invoke + * the decorated parser) to implement extra decoration. + */ + public String parse(InputStream stream, Iterable contents) + throws IOException, TikaException { + return parser.parse(stream, contents); + } + +} Property changes on: src\main\java\org\apache\tika\parser\ParserDecorator.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/main/java/org/apache/tika/parser/msexcel/MsExcelParser.java =================================================================== --- src/main/java/org/apache/tika/parser/msexcel/MsExcelParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/msexcel/MsExcelParser.java (working copy) @@ -27,9 +27,9 @@ /** * Excel parser */ -public class MsExcelParser extends Parser { +public class MsExcelParser implements Parser { - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { try { MSExtractor extractor = new ExcelExtractor(); Index: src/main/java/org/apache/tika/parser/opendocument/OpenOfficeParser.java =================================================================== --- src/main/java/org/apache/tika/parser/opendocument/OpenOfficeParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/opendocument/OpenOfficeParser.java (working copy) @@ -42,7 +42,7 @@ /** * OpenOffice parser */ -public class OpenOfficeParser extends Parser { +public class OpenOfficeParser implements Parser { static Logger logger = Logger.getRootLogger(); private final Namespace NS_DC = Namespace.getNamespace("dc", @@ -75,7 +75,7 @@ return xmlDoc; } - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { Document xmlDoc = parse(stream); XMLParser xp = new XMLParser(); Index: src/main/java/org/apache/tika/parser/xml/XMLParser.java =================================================================== --- src/main/java/org/apache/tika/parser/xml/XMLParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/xml/XMLParser.java (working copy) @@ -44,13 +44,23 @@ /** * XML parser */ -public class XMLParser extends Parser { +public class XMLParser implements Parser { static Logger logger = Logger.getRootLogger(); private SimpleNamespaceContext nsc = new SimpleNamespaceContext(); - protected String parse(InputStream stream, Iterable contents) + private String namespace; + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { Document xmlDoc = Utils.parse(stream); if (exist(getAllDocumentNs(xmlDoc), getNamespace())) { Index: src/main/java/org/apache/tika/parser/ParserPostProcessor.java =================================================================== --- src/main/java/org/apache/tika/parser/ParserPostProcessor.java (revision 0) +++ src/main/java/org/apache/tika/parser/ParserPostProcessor.java (revision 0) @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import org.apache.log4j.Logger; +import org.apache.oro.text.regex.MalformedPatternException; +import org.apache.tika.config.Content; +import org.apache.tika.exception.TikaException; +import org.apache.tika.utils.RegexUtils; + +/** + * Parser decorator that post-processes the results from a decorated parser. + * The post-processing takes care of filling in any "fulltext", "summary", and + * regexp {@link Content} objects with the full text content returned by + * the decorated parser. The post-processing also catches and logs any + * exceptions thrown by the decorated parser. + */ +public class ParserPostProcessor extends ParserDecorator { + + /** + * Logger instance. + */ + private static final Logger logger = + Logger.getLogger(ParserPostProcessor.class); + + /** + * Creates a post-processing decorator for the given parser. + * + * @param parser the parser to be decorated + */ + public ParserPostProcessor(Parser parser) { + super(parser); + } + + /** + * Forwards the call to the delegated parser and post-processes the + * results as described above. + */ + public String parse(InputStream stream, Iterable contents) + throws IOException, TikaException { + try { + String contentStr = super.parse(stream, contents); + + for (Content content : contents) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if ("summary".equalsIgnoreCase(content.getTextSelect())) { + int length = Math.min(contentStr.length(), 500); + String summary = contentStr.substring(0, length); + content.setValue(summary); + } else if (content.getRegexSelect() != null) { + String regex = content.getRegexSelect(); + try { + List values = + RegexUtils.extract(contentStr, regex); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues( + values.toArray(new String[values.size()])); + } + } catch (MalformedPatternException e) { + logger.error( + "Invalid regular expression: " + regex, e); + } + } + } + + return contentStr; + } catch (Exception e) { + logger.error("Parse error: " + e.getMessage(), e); + return ""; + } + } + +} Property changes on: src\main\java\org\apache\tika\parser\ParserPostProcessor.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/main/java/org/apache/tika/parser/html/HtmlParser.java =================================================================== --- src/main/java/org/apache/tika/parser/html/HtmlParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/html/HtmlParser.java (working copy) @@ -33,11 +33,11 @@ * Html parser * */ -public class HtmlParser extends Parser { +public class HtmlParser implements Parser { static Logger logger = Logger.getRootLogger(); - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { Tidy tidy = new Tidy(); tidy.setQuiet(true); Index: src/main/java/org/apache/tika/parser/ParserFactory.java =================================================================== --- src/main/java/org/apache/tika/parser/ParserFactory.java (revision 582120) +++ src/main/java/org/apache/tika/parser/ParserFactory.java (working copy) @@ -20,6 +20,7 @@ import org.apache.log4j.Logger; import org.apache.tika.config.ParserConfig; import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.xml.XMLParser; /** * Factory class. Build parser from xml config file. @@ -38,7 +39,12 @@ } try { logger.info("Loading parser class = " + className); - return (Parser) Class.forName(className).newInstance(); + Parser parser = (Parser) Class.forName(className).newInstance(); + // FIXME: Replace with proper JavaBean dependency/config injection + if (parser instanceof XMLParser) { + ((XMLParser) parser).setNamespace(config.getNameSpace()); + } + return new ParserPostProcessor(parser); } catch (Exception e) { logger.error("Unable to instantiate parser: " + className, e); throw new TikaException(e.getMessage()); Index: src/main/java/org/apache/tika/parser/mspowerpoint/MsPowerPointParser.java =================================================================== --- src/main/java/org/apache/tika/parser/mspowerpoint/MsPowerPointParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/mspowerpoint/MsPowerPointParser.java (working copy) @@ -27,10 +27,10 @@ /** * Power point parser */ -public class MsPowerPointParser extends Parser { +public class MsPowerPointParser implements Parser { - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { try { MSExtractor extractor = new PPTExtractor(); Index: src/main/java/org/apache/tika/parser/pdf/PDFParser.java =================================================================== --- src/main/java/org/apache/tika/parser/pdf/PDFParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/pdf/PDFParser.java (working copy) @@ -32,9 +32,9 @@ /** * PDF parser */ -public class PDFParser extends Parser { +public class PDFParser implements Parser { - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { try { PDDocument pdfDocument = PDDocument.load(stream); Index: src/main/java/org/apache/tika/parser/msword/MsWordParser.java =================================================================== --- src/main/java/org/apache/tika/parser/msword/MsWordParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/msword/MsWordParser.java (working copy) @@ -27,9 +27,9 @@ /** * Word parser */ -public class MsWordParser extends Parser { +public class MsWordParser implements Parser { - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { try { MSExtractor extractor = new WordExtractor(); Index: src/main/java/org/apache/tika/parser/rtf/RTFParser.java =================================================================== --- src/main/java/org/apache/tika/parser/rtf/RTFParser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/rtf/RTFParser.java (working copy) @@ -29,9 +29,9 @@ /** * RTF parser */ -public class RTFParser extends Parser { +public class RTFParser implements Parser { - protected String parse(InputStream stream, Iterable contents) + public String parse(InputStream stream, Iterable contents) throws IOException, TikaException { try { DefaultStyledDocument sd = new DefaultStyledDocument(); Index: src/main/java/org/apache/tika/parser/Parser.java =================================================================== --- src/main/java/org/apache/tika/parser/Parser.java (revision 582120) +++ src/main/java/org/apache/tika/parser/Parser.java (working copy) @@ -18,85 +18,30 @@ import java.io.IOException; import java.io.InputStream; -import java.util.List; -import java.util.Map; -import org.apache.log4j.Logger; -import org.apache.oro.text.regex.MalformedPatternException; import org.apache.tika.config.Content; import org.apache.tika.exception.TikaException; -import org.apache.tika.utils.RegexUtils; /** - * Abstract class Parser + * Tika parser interface */ -public abstract class Parser { +public interface Parser { - private static final Logger logger = Logger.getLogger(Parser.class); - - private String mimeType; - - private String namespace; - /** - * Get document mime type + * Parses a document from the given input stream and returns the + * extracted full text content of the document. Fills in selected + * metadata information in the given set of {@link Content} instances. + *

+ * The given stream is consumed but not closed by this method. + * The responsibility to close the stream remains on the caller. + * + * @param stream the document to be parsed + * @param contents set of metadata information to extract + * @return full text content of the document + * @throws IOException if the document could not be read + * @throws TikaException if the document could not be parsed */ - public String getMimeType() { - return mimeType; - } - - /** - * Set document mime type - */ - public void setMimeType(String mimeType) { - this.mimeType = mimeType; - } - - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public String getContents(InputStream stream, Map contents) { - try { - String contentStr = parse(stream, contents.values()); - - for (Content content : contents.values()) { - if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { - content.setValue(contentStr); - } else if ("summary".equalsIgnoreCase(content.getTextSelect())) { - int length = Math.min(contentStr.length(), 500); - String summary = contentStr.substring(0, length); - content.setValue(summary); - } else if (content.getRegexSelect() != null) { - String regex = content.getRegexSelect(); - try { - List values = - RegexUtils.extract(contentStr, regex); - if (values.size() > 0) { - content.setValue(values.get(0)); - content.setValues( - values.toArray(new String[values.size()])); - } - } catch (MalformedPatternException e) { - logger.error( - "Invalid regular expression: " + regex, e); - } - } - } - - return contentStr; - } catch (Exception e) { - logger.error("Parse error: " + e.getMessage(), e); - return ""; - } - } - - protected abstract String parse( - InputStream stream, Iterable contents) + String parse(InputStream stream, Iterable contents) throws IOException, TikaException; }