Index: src/main/java/org/apache/tika/utils/Utils.java =================================================================== --- src/main/java/org/apache/tika/utils/Utils.java (revision 578143) +++ src/main/java/org/apache/tika/utils/Utils.java (working copy) @@ -29,9 +29,9 @@ import java.io.StringWriter; import java.io.Writer; import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -53,26 +53,27 @@ static Logger logger = Logger.getRootLogger(); - public static String toString(Collection structuredContent) { + public static String toString(Map structuredContent) { final StringWriter sw = new StringWriter(); print(structuredContent,sw); return sw.toString(); } - public static void print(Collection structuredContent) { + public static void print(Map structuredContent) { print(structuredContent,new OutputStreamWriter(System.out)); } - public static void print(Collection structuredContent,Writer outputWriter) { + public static void print( + Map structuredContent, Writer outputWriter) { + structuredContent = new TreeMap(structuredContent); final PrintWriter output = new PrintWriter(outputWriter,true); - for (Iterator iter = structuredContent.iterator(); iter - .hasNext();) { - Content ct = iter.next(); + for (Map.Entry entry : structuredContent.entrySet()) { + Content ct = entry.getValue(); if (ct.getValue() != null) { - output.print(ct.getName() + ": "); + output.print(entry.getKey() + ": "); output.println(ct.getValue()); } else if (ct.getValues() != null) { - output.print(ct.getName() + ": "); + output.print(entry.getKey() + ": "); for (int j = 0; j < ct.getValues().length; j++) { if (j == 0) output.println(ct.getValues()[j]); @@ -82,7 +83,7 @@ } } else { // there are no values, but there is a Content object System.out.println( - "Content '" + ct.getName() + "' has no values."); + "Content '" + entry.getKey() + "' has no values."); } } } Index: src/main/java/org/apache/tika/utils/MSExtractor.java =================================================================== --- src/main/java/org/apache/tika/utils/MSExtractor.java (revision 578143) +++ src/main/java/org/apache/tika/utils/MSExtractor.java (working copy) @@ -18,9 +18,7 @@ // JDK imports import java.io.InputStream; -import java.util.Date; -import java.util.List; -import java.util.Properties; +import java.util.Map; import org.apache.tika.config.Content; // Jakarta POI imports @@ -43,14 +41,10 @@ private String text = null; private POIFSReader reader = null; - - private List contents; - /** Constructs a new Microsoft document extractor. */ - public MSExtractor() { - } - - public void setContents(List contents){ + private Map contents; + + public void setContents(Map contents) { this.contents = contents; } @@ -61,7 +55,8 @@ // First, extract properties this.reader = new POIFSReader(); - this.reader.registerListener(new PropertiesReaderListener(contents), + this.reader.registerListener( + new PropertiesReaderListener(), SummaryInformation.DEFAULT_STREAM_NAME); //input.reset(); if (input.available() > 0) { @@ -86,12 +81,7 @@ } private class PropertiesReaderListener implements POIFSReaderListener { - private List contents; - PropertiesReaderListener(List contents) { - this.contents = contents; - } - public void processPOIFSReaderEvent(POIFSReaderEvent event) { if (!event.getName().startsWith( SummaryInformation.DEFAULT_STREAM_NAME)) { @@ -102,8 +92,8 @@ SummaryInformation si = (SummaryInformation) PropertySetFactory .create(event.getStream()); - for (int i = 0; i < contents.size(); i++) { - Content content = contents.get(i); + for (Map.Entry entry : contents.entrySet()) { + Content content = entry.getValue(); if (content.getTextSelect().equalsIgnoreCase("title")) { content.setValue(si.getTitle()); } @@ -134,35 +124,11 @@ else if (content.getTextSelect().equalsIgnoreCase("creationdate")) { content.setValue(si.getCreateDateTime().toString()); } - else if (content.getTextSelect().equalsIgnoreCase("")) { - //content.setValue(si.getCharCount()); - } - else if (content.getTextSelect().equals("")) { - - } - else if (content.getTextSelect().equals("")) { - - } - else if (content.getTextSelect().equals("")) { - - } - else if (content.getTextSelect().equals("")) { - - } - else if (content.getTextSelect().equals("")) { - - } - else if (content.getTextSelect().equals("")) { - - } - System.out.println(content.getName()+" :"+content.getValue()); + System.out.println(entry.getKey() + ": " + content.getValue()); } - } catch (Exception ex) { } - } - } } Index: src/main/java/org/apache/tika/config/LiusConfig.java =================================================================== --- src/main/java/org/apache/tika/config/LiusConfig.java (revision 578143) +++ src/main/java/org/apache/tika/config/LiusConfig.java (working copy) @@ -122,14 +122,14 @@ mimes.put(mime, null); } pc.setMimes(mimes); - List contents = new ArrayList(); + Map contents = new HashMap(); if (parserElem.getChild("extract") != null) { - List contentsElems = parserElem.getChild("extract") - .getChildren(); + List contentsElems = + parserElem.getChild("extract").getChildren(); for (int j = 0; j < contentsElems.size(); j++) { Content content = new Content(); Element contentElem = (Element) contentsElems.get(j); - content.setName(contentElem.getAttributeValue("name")); + String name = contentElem.getAttributeValue("name"); if (contentElem.getAttribute("xpathSelect") != null) { content.setXPathSelect(contentElem .getAttributeValue("xpathSelect")); @@ -142,7 +142,7 @@ content.setRegexSelect(contentElem.getChild( "regexSelect").getTextTrim()); } - contents.add(content); + contents.put(name, content); } } pc.setContents(contents); Index: src/main/java/org/apache/tika/config/ParserConfig.java =================================================================== --- src/main/java/org/apache/tika/config/ParserConfig.java (revision 578143) +++ src/main/java/org/apache/tika/config/ParserConfig.java (working copy) @@ -16,7 +16,6 @@ */ package org.apache.tika.config; -import java.util.List; import java.util.Map; /** @@ -33,13 +32,13 @@ private String nameSpace; - private List contents; + private Map contents; - public List getContents() { + public Map getContents() { return contents; } - public void setContents(List contents) { + public void setContents(Map contents) { this.contents = contents; } Index: src/main/java/org/apache/tika/config/Content.java =================================================================== --- src/main/java/org/apache/tika/config/Content.java (revision 578143) +++ src/main/java/org/apache/tika/config/Content.java (working copy) @@ -21,21 +21,13 @@ * @author Rida Benjelloun (ridabenjelloun@apache.org) */ public class Content { - - private String name; + private String value; private String[] values; private String textSelect; private String xPathSelect; private String regexSelect; - - - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; - } + public String getRegexSelect() { return regexSelect; } Index: src/main/java/org/apache/tika/parser/txt/TXTParser.java =================================================================== --- src/main/java/org/apache/tika/parser/txt/TXTParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/txt/TXTParser.java (working copy) @@ -17,11 +17,8 @@ package org.apache.tika.parser.txt; import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,75 +36,59 @@ */ public class TXTParser extends Parser { - static Logger logger = Logger.getRootLogger(); + private static final Logger logger = Logger.getLogger(TXTParser.class); - private Map contentsMap; + private boolean parsed = false; private String contentStr; - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } - - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - if (ct.getTextSelect() != null) { - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - } - - } else if (ct.getRegexSelect() != null) { + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + try { + InputStream stream = getInputStream(); try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + StringBuilder builder = new StringBuilder(); + BufferedReader reader = + new BufferedReader(new InputStreamReader(stream)); + String line = reader.readLine(); + while (line != null) { + builder.append(line); + builder.append(" "); + line = reader.readLine(); } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); + contentStr = builder.toString(); + + for (Content content : contents.values()) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); + } + } + } + } finally { + stream.close(); } + } catch (Exception e) { + logger.error(e.getMessage()); } - contentsMap.put(ct.getName(), ct); - } - return ctt; - + parsed = true; + } + return contents; } - @Override public String getStrContent() { - StringBuffer sb = new StringBuffer(); - try { - BufferedReader br = new BufferedReader(new InputStreamReader( - getInputStream())); - String line = null; - while ((line = br.readLine()) != null) { - sb.append(line); - sb.append(" "); - } - } catch (FileNotFoundException ex) { - logger.error(ex.getMessage()); - } catch (IOException ex1) { - logger.error(ex1.getMessage()); - } finally { - try { - getInputStream().close(); - } catch (IOException e) { - logger.error(e.getMessage()); - } - } - contentStr = sb.toString(); + getContents(); return contentStr; } Index: src/main/java/org/apache/tika/parser/msexcel/MsExcelParser.java =================================================================== --- src/main/java/org/apache/tika/parser/msexcel/MsExcelParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/msexcel/MsExcelParser.java (working copy) @@ -16,8 +16,6 @@ */ package org.apache.tika.parser.msexcel; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -35,61 +33,49 @@ * @author Rida Benjelloun (ridabenjelloun@apache.org) */ public class MsExcelParser extends Parser { - private MSExtractor extrator = new ExcelExtractor(); - private String contentStr; + private static final Logger logger = Logger.getLogger(MsExcelParser.class); - private Map contentsMap; + private boolean parsed = false; - static Logger logger = Logger.getRootLogger(); + private String contentStr; - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + try { + MSExtractor extractor = new ExcelExtractor(); + extractor.setContents(contents); + contentStr = extractor.extractText(getInputStream()); - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - if (ct.getTextSelect() != null) { - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - } - - } else if (ct.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + for (Content content : contents.values()) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error("Invalid regexp", e); + } } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } + } catch (Exception e) { + logger.error("Parse error", e);; } - contentsMap.put(ct.getName(), ct); - } - return ctt; + parsed = true; + } + return contents; } public String getStrContent() { - // extrator.setContents(getParserConfig().getContents()); - try { - contentStr = extrator.extractText(getInputStream()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + getContents(); return contentStr; } + } Index: src/main/java/org/apache/tika/parser/opendocument/OpenOfficeParser.java =================================================================== --- src/main/java/org/apache/tika/parser/opendocument/OpenOfficeParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/opendocument/OpenOfficeParser.java (working copy) @@ -23,8 +23,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; @@ -49,34 +47,31 @@ * @author Rida Benjelloun (ridabenjelloun@apache.org) */ public class OpenOfficeParser extends Parser { - static Logger logger = Logger.getRootLogger(); + private static final Logger logger = Logger.getLogger(OpenOfficeParser.class); + + private boolean parsed = false; + private final Namespace NS_DC = Namespace.getNamespace("dc", "http://purl.org/dc/elements/1.1/"); - private XMLParser xp = new XMLParser(); - - private org.jdom.Document xmlDoc; - - private Map contentsMap; - private String contentStr; public org.jdom.Document parse(InputStream is) { - xmlDoc = new org.jdom.Document(); + org.jdom.Document xmlDoc = new org.jdom.Document(); org.jdom.Document xmlMeta = new org.jdom.Document(); try { - List files = unzip(is); + List files = unzip(is); SAXBuilder builder = new SAXBuilder(); builder.setEntityResolver(new OpenOfficeEntityResolver()); builder.setValidation(false); - xmlDoc = builder.build((InputStream) files.get(0)); - xmlMeta = builder.build((InputStream) files.get(1)); + xmlDoc = builder.build(files.get(0)); + xmlMeta = builder.build(files.get(1)); Element rootMeta = xmlMeta.getRootElement(); Element meta = null; - List ls = new ArrayList(); - if ((ls = rootMeta.getChildren()).size() > 0) { + List ls = rootMeta.getChildren(); + if (ls.size() > 0) { meta = (Element) ls.get(0); } xmlDoc.getRootElement().addContent(meta.detach()); @@ -90,47 +85,46 @@ return xmlDoc; } - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - if (xmlDoc == null) - xmlDoc = Utils.parse(getInputStream()); - List documentNs = xp.getAllDocumentNs(xmlDoc); - List ctt = super.getContents(); - Iterator it = ctt.iterator(); - contentsMap = new HashMap(); + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + try { + org.jdom.Document xmlDoc = parse(getInputStream()); + XMLParser xp = new XMLParser(); + contentStr = xp.concatOccurance(xmlDoc, "//*", " "); - while (it.hasNext()) { - Content content = (Content) it.next(); - if (content.getXPathSelect() != null) { - xp.extractContent(xmlDoc, content, contentsMap); - } else if (content.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, - content.getRegexSelect()); - if (valuesLs.size() > 0) { - content.setValue(valuesLs.get(0)); - content.setValues(valuesLs.toArray(new String[0])); + for (Content content : contents.values()) { + if (content.getXPathSelect() != null) { + xp.extractContent(xmlDoc, content); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); + } } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } + } catch (Exception e) { + logger.error("Parse error", e); } - } - return ctt; + parsed = true; + } + return contents; } public String getStrContent() { - if (xmlDoc == null) - xmlDoc = parse(getInputStream()); - contentStr = xp.concatOccurance(xmlDoc, "//*", " "); + getContents(); return contentStr; } - public List unzip(InputStream is) { - List res = new ArrayList(); + public List unzip(InputStream is) { + List res = new ArrayList(); try { ZipInputStream in = new ZipInputStream(is); ZipEntry entry = null; @@ -155,13 +149,6 @@ return res; } - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } - protected void copyInputStream(InputStream in, OutputStream out) throws IOException { byte[] buffer = new byte[1024]; Index: src/main/java/org/apache/tika/parser/xml/XMLParser.java =================================================================== --- src/main/java/org/apache/tika/parser/xml/XMLParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/xml/XMLParser.java (working copy) @@ -17,7 +17,6 @@ package org.apache.tika.parser.xml; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -48,61 +47,49 @@ * @author Rida Benjelloun (ridabenjelloun@apache.org) */ public class XMLParser extends Parser { - static Logger logger = Logger.getRootLogger(); - private Document xmlDoc = null; + private static final Logger logger = Logger.getLogger(XMLParser.class); + private boolean parsed = false; + private SimpleNamespaceContext nsc = new SimpleNamespaceContext(); - private Map contentsMap; - private String contentStr; - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } - public String getStrContent() { - if (xmlDoc == null) - xmlDoc = Utils.parse(getInputStream()); - contentStr = concatOccurance(xmlDoc, "//*", " "); + getContents(); return contentStr; } - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - if (xmlDoc == null) - xmlDoc = Utils.parse(getInputStream()); - List documentNs = getAllDocumentNs(xmlDoc); - List ctt = super.getContents(); - Iterator it = ctt.iterator(); - contentsMap = new HashMap(); - if (exist(documentNs, getNamespace())) { - while (it.hasNext()) { - Content content = (Content) it.next(); - if (content.getXPathSelect() != null) { - extractContent(xmlDoc, content, contentsMap); - } else if (content.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, - content.getRegexSelect()); - if (valuesLs.size() > 0) { - content.setValue(valuesLs.get(0)); - content.setValues(valuesLs.toArray(new String[0])); + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + Document xmlDoc = Utils.parse(getInputStream()); + contentStr = concatOccurance(xmlDoc, "//*", " "); + + List documentNs = getAllDocumentNs(xmlDoc); + if (exist(documentNs, getNamespace())) { + for (Content content : contents.values()) { + if (content.getXPathSelect() != null) { + extractContent(xmlDoc, content); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } } } - } - return ctt; + parsed = true; + } + return contents; } public String concatOccurance(Object xmlDoc, String xpath, String concatSep) { @@ -158,8 +145,8 @@ return chaineConcat.toString().trim(); } - public List getAllDocumentNs(org.jdom.Document doc) { - List ls = new ArrayList(); + public List getAllDocumentNs(org.jdom.Document doc) { + List ls = new ArrayList(); processChildren(doc.getRootElement(), ls); return ls; } @@ -175,7 +162,7 @@ return false; } - private void processChildren(Element elem, List ns) { + private void processChildren(Element elem, List ns) { Namespace nsCourent = (Namespace) elem.getNamespace(); String nsUri = (nsCourent.getURI()); if (!exist(ns, nsUri)) { @@ -201,8 +188,7 @@ } } - public void extractContent(Document xmlDoc, Content content, - Map contentsMap) { + public void extractContent(Document xmlDoc, Content content) { try { JDOMXPath xp = new JDOMXPath(content.getXPathSelect()); xp.setNamespaceContext(nsc); @@ -243,7 +229,6 @@ if (values.length > 0) { content.setValue(values[0]); content.setValues(values); - contentsMap.put(content.getName(), content); } } catch (JaxenException e) { logger.error(e.getMessage()); Index: src/main/java/org/apache/tika/parser/html/HtmlParser.java =================================================================== --- src/main/java/org/apache/tika/parser/html/HtmlParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/html/HtmlParser.java (working copy) @@ -17,8 +17,6 @@ package org.apache.tika.parser.html; import java.io.InputStream; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -40,62 +38,44 @@ */ public class HtmlParser extends Parser { - static Logger logger = Logger.getRootLogger(); + private static final Logger logger = Logger.getLogger(HtmlParser.class); - private Node root = null; + private boolean parsed = false; private String contentStr; - private Map contentsMap; + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + Node root = getRoot(getInputStream()); + contentStr = getTextContent(root); - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } - - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - if (ct.getTextSelect() != null) { - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - } else { - extractElementTxt((Element) root, ct); - } - - } - - else if (ct.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + for (Content content : contents.values()) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if (content.getTextSelect() != null) { + extractElementTxt((Element) root, content); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } } - contentsMap.put(ct.getName(), ct); - } - return ctt; - + parsed = true; + } + return contents; } public String getStrContent() { - if (root == null) - root = getRoot(getInputStream()); - contentStr = getTextContent(root); + getContents(); return contentStr; } @@ -108,8 +88,7 @@ } private void extractElementTxt(Element root, Content content) { - - NodeList children = root.getElementsByTagName(content.getName()); + NodeList children = root.getElementsByTagName(content.getTextSelect()); if (children != null) { if (children.getLength() > 0) { if (children.getLength() == 1) { Index: src/main/java/org/apache/tika/parser/mspowerpoint/MsPowerPointParser.java =================================================================== --- src/main/java/org/apache/tika/parser/mspowerpoint/MsPowerPointParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/mspowerpoint/MsPowerPointParser.java (working copy) @@ -16,8 +16,6 @@ */ package org.apache.tika.parser.mspowerpoint; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -35,69 +33,47 @@ */ public class MsPowerPointParser extends Parser { - private PPTExtractor extrator = new PPTExtractor(); + private static final Logger logger = Logger.getLogger(MsPowerPointParser.class); + private boolean parsed = false; + private String contentStr; - private Map contentsMap; + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + try { + PPTExtractor extractor = new PPTExtractor(); + extractor.setContents(contents); + contentStr = extractor.extractText(getInputStream()); - static Logger logger = Logger.getRootLogger(); - - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } - - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - if (ct.getTextSelect() != null) { - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - } - - } else if (ct.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + for (Content content : contents.values()) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); + } } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } + } catch (Exception e) { + logger.error("Parse error", e); } - contentsMap.put(ct.getName(), ct); - } - return ctt; + parsed = true; + } + return contents; } - /* - * public List getContents() { - * extrator.setContents(getParserConfig().getContents()); try { - * extrator.extract(getInputStream()); } catch (Exception e) { // TODO - * Auto-generated catch block e.printStackTrace(); } return - * getParserConfig().getContents(); } - */ - public String getStrContent() { - extrator.setContents(super.getContents()); - try { - contentStr = extrator.extractText(getInputStream()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + getContents(); return contentStr; } Index: src/main/java/org/apache/tika/parser/pdf/PDFParser.java =================================================================== --- src/main/java/org/apache/tika/parser/pdf/PDFParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/pdf/PDFParser.java (working copy) @@ -18,8 +18,7 @@ import java.io.IOException; import java.io.StringWriter; -import java.util.HashMap; -import java.util.Iterator; +import java.util.Calendar; import java.util.List; import java.util.Map; @@ -41,156 +40,118 @@ * @author Rida Benjelloun (ridabenjelloun@apache.org) */ public class PDFParser extends Parser { - static Logger logger = Logger.getRootLogger(); - private String contentStr = ""; + private static final Logger logger = Logger.getLogger(PDFParser.class); - private PDDocument pdfDocument = null; + private boolean parsed = false; - private Map contentsMap; + private String contentStr = ""; public String getStrContent() { - - try { - pdfDocument = PDDocument.load(getInputStream()); - if (pdfDocument.isEncrypted()) { - pdfDocument.decrypt(""); - } - StringWriter writer = new StringWriter(); - PDFTextStripper stripper = new PDFTextStripper(); - stripper.writeText(pdfDocument, writer); - contentStr = writer.getBuffer().toString(); - } catch (CryptographyException e) { - logger.error(e.getMessage()); - } catch (IOException e) { - e.printStackTrace(); - logger.error(e.getMessage()); - } catch (InvalidPasswordException e) { - logger.error(e.getMessage()); - } finally { - if (pdfDocument != null) { - try { - pdfDocument.close(); - } catch (IOException ex) { - logger.error(ex.getMessage()); - } - } - } + getContents(); return contentStr; } - public List getContents() { + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + PDDocument pdf = null; + try { + pdf = PDDocument.load(getInputStream()); + if (pdf.isEncrypted()) { + pdf.decrypt(""); + } + StringWriter writer = new StringWriter(); + new PDFTextStripper().writeText(pdf, writer); + contentStr = writer.getBuffer().toString(); - // String contents = getContent(); - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - - if (ct.getTextSelect() != null) { - - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - - } else { - try { - PDDocumentInformation metaData = pdfDocument - .getDocumentInformation(); - if (ct.getTextSelect().equalsIgnoreCase("title")) { - if (metaData.getTitle() != null) { - ct.setValue(metaData.getTitle()); - + for (Content content : contents.values()) { + String text = content.getTextSelect(); + if ("fulltext".equalsIgnoreCase(text)) { + content.setValue(contentStr); + } else if (text != null) { + try { + PDDocumentInformation metaData = + pdf.getDocumentInformation(); + if (text.equalsIgnoreCase("title")) { + if (metaData.getTitle() != null) { + content.setValue(metaData.getTitle()); + } + } else if (text.equalsIgnoreCase("author")) { + if (metaData.getAuthor() != null) { + content.setValue(metaData.getAuthor()); + } + } else if (text.equalsIgnoreCase("creator")) { + if (metaData.getCreator() != null) { + content.setValue(metaData.getCreator()); + } + } else if (text.equalsIgnoreCase("keywords")) { + if (metaData.getKeywords() != null) { + content.setValue(metaData.getKeywords()); + } + } else if (text.equalsIgnoreCase("producer")) { + if (metaData.getProducer() != null) { + content.setValue(metaData.getProducer()); + } + } else if (text.equalsIgnoreCase("subject")) { + if (metaData.getSubject() != null) { + content.setValue(metaData.getSubject()); + } + } else if (text.equalsIgnoreCase("trapped")) { + if (metaData.getTrapped() != null) { + content.setValue(metaData.getTrapped()); + } + } else if (text.equalsIgnoreCase("creationDate")) { + Calendar date = metaData.getCreationDate(); + if (date != null) { + content.setValue(date.getTime().toString()); + } + } else if (text.equalsIgnoreCase("modificationDate")) { + Calendar date = metaData.getModificationDate(); + if (date != null) { + content.setValue(date.getTime().toString()); + } + } else if (text.equalsIgnoreCase("summary")) { + String summary = contentStr.substring( + 0, Math.min(contentStr.length(), 500)); + content.setValue(summary); } - } else if (ct.getTextSelect() - .equalsIgnoreCase("author")) { - if (metaData.getAuthor() != null) { - ct.setValue(metaData.getAuthor()); - + } catch (IOException e) { + logger.error(e.getMessage()); + } + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); } - } else if (ct.getTextSelect().equalsIgnoreCase( - "creator")) { - if (metaData.getCreator() != null) { - ct.setValue(metaData.getCreator()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "keywords")) { - if (metaData.getKeywords() != null) { - ct.setValue(metaData.getKeywords()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "producer")) { - if (metaData.getProducer() != null) { - ct.setValue(metaData.getProducer()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "subject")) { - if (metaData.getSubject() != null) { - ct.setValue(metaData.getSubject()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "trapped")) { - if (metaData.getTrapped() != null) { - ct.setValue(metaData.getTrapped()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "creationDate")) { - if (metaData.getCreationDate() != null) { - ct.setValue(metaData.getCreationDate() - .getTime().toString()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "modificationDate")) { - if (metaData.getModificationDate() != null) { - ct.setValue(metaData.getModificationDate() - .getTime().toString()); - - } - } else if (ct.getTextSelect().equalsIgnoreCase( - "summary")) { - int summarySize = Math - .min(contentStr.length(), 500); - String summary = contentStr.substring(0, - summarySize); - ct.setValue(summary); + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); } - } catch (IOException e) { - logger.error(e.getMessage()); } } - - } else if (ct.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + } catch (CryptographyException e) { + logger.error(e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + logger.error(e.getMessage()); + } catch (InvalidPasswordException e) { + logger.error(e.getMessage()); + } finally { + if (pdf != null) { + try { + pdf.close(); + } catch (IOException ex) { + logger.error(ex.getMessage()); } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } } - contentsMap.put(ct.getName(), ct); - } - return ctt; - } - - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); + parsed =true; } - return contentsMap.get(name); + return contents; } } Index: src/main/java/org/apache/tika/parser/msword/MsWordParser.java =================================================================== --- src/main/java/org/apache/tika/parser/msword/MsWordParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/msword/MsWordParser.java (working copy) @@ -36,62 +36,47 @@ */ public class MsWordParser extends Parser { - private MSExtractor extractor = new WordExtractor(); + private static final Logger logger = Logger.getLogger(MsWordParser.class); + private boolean parsed = false; + private String contentStr; - private Map contentsMap; + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + try { + MSExtractor extractor = new WordExtractor(); + extractor.setContents(contents); + contentStr = extractor.extractText(getInputStream()); - static Logger logger = Logger.getRootLogger(); - - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } - - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - if (ct.getTextSelect() != null) { - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - } - - } else if (ct.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + for (Content content : contents.values()) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); + } } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } + } catch (Exception e) { + logger.error("Parse error", e); } - contentsMap.put(ct.getName(), ct); - } - return ctt; - + parsed = true; + } + return contents; } public String getStrContent() { - // extractor - try { - contentStr = extractor.extractText(getInputStream()); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + getContents(); return contentStr; } Index: src/main/java/org/apache/tika/parser/rtf/RTFParser.java =================================================================== --- src/main/java/org/apache/tika/parser/rtf/RTFParser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/rtf/RTFParser.java (working copy) @@ -16,13 +16,9 @@ */ package org.apache.tika.parser.rtf; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import javax.swing.text.BadLocationException; import javax.swing.text.DefaultStyledDocument; import javax.swing.text.rtf.RTFEditorKit; @@ -40,63 +36,48 @@ */ public class RTFParser extends Parser { - static Logger logger = Logger.getRootLogger(); + private static final Logger logger = Logger.getLogger(RTFParser.class); - private Map contentsMap; + private boolean parsed = false; private String contentStr; - public Content getContent(String name) { - if (contentsMap == null || contentsMap.isEmpty()) { - getContents(); - } - return contentsMap.get(name); - } + public Map getContents() { + Map contents = super.getContents(); + if (!parsed) { + try { + DefaultStyledDocument sd = new DefaultStyledDocument(); + RTFEditorKit kit = new RTFEditorKit(); + kit.read(getInputStream(), sd, 0); + contentStr = sd.getText(0, sd.getLength()); - public List getContents() { - if (contentStr == null) { - contentStr = getStrContent(); - } - List ctt = super.getContents(); - contentsMap = new HashMap(); - Iterator i = ctt.iterator(); - while (i.hasNext()) { - Content ct = (Content) i.next(); - if (ct.getTextSelect() != null) { - if (ct.getTextSelect().equalsIgnoreCase("fulltext")) { - ct.setValue(contentStr); - } - - } else if (ct.getRegexSelect() != null) { - try { - List valuesLs = RegexUtils.extract(contentStr, ct - .getRegexSelect()); - if (valuesLs.size() > 0) { - ct.setValue(valuesLs.get(0)); - ct.setValues(valuesLs.toArray(new String[0])); + for (Content content : contents.values()) { + if ("fulltext".equalsIgnoreCase(content.getTextSelect())) { + content.setValue(contentStr); + } else if (content.getRegexSelect() != null) { + try { + List values = RegexUtils.extract( + contentStr, content.getRegexSelect()); + if (values.size() > 0) { + content.setValue(values.get(0)); + content.setValues(values.toArray(new String[0])); + } + } catch (MalformedPatternException e) { + logger.error(e.getMessage()); + } } - } catch (MalformedPatternException e) { - logger.error(e.getMessage()); } + } catch (Exception e) { + logger.error(e.getMessage()); } - contentsMap.put(ct.getName(), ct); - } - return ctt; + parsed = true; + } + return contents; } - @Override public String getStrContent() { - try { - DefaultStyledDocument sd = new DefaultStyledDocument(); - RTFEditorKit kit = new RTFEditorKit(); - kit.read(getInputStream(), sd, 0); - contentStr = sd.getText(0, sd.getLength()); - } catch (IOException e) { - logger.error(e.getMessage()); - } catch (BadLocationException j) { - logger.error(j.getMessage()); - } + getContents(); return contentStr; } Index: src/main/java/org/apache/tika/parser/Parser.java =================================================================== --- src/main/java/org/apache/tika/parser/Parser.java (revision 578157) +++ src/main/java/org/apache/tika/parser/Parser.java (working copy) @@ -17,7 +17,7 @@ package org.apache.tika.parser; import java.io.InputStream; -import java.util.List; +import java.util.Map; import org.apache.tika.config.Content; @@ -34,7 +34,7 @@ private String namespace; - private List contents; + private Map contents; public void setInputStream(InputStream is) { this.is = is; @@ -76,18 +76,20 @@ * It could be a document metadata, XPath selection, regex selection or * fulltext */ - public abstract Content getContent(String name); + public Content getContent(String name) { + return getContents().get(name); + } /** * Get a List of contents objects, this objects are configured from the * LiusConfig Xml file. It could be a document metadata, XPath selection, * regex selection or fulltext */ - public List getContents() { + public Map getContents() { return contents; } - public void setContents(List contents) { + public void setContents(Map contents) { this.contents = contents; }