Index: conf/nutch-default.xml =================================================================== --- conf/nutch-default.xml (revision 505236) +++ conf/nutch-default.xml (working copy) @@ -788,8 +788,15 @@ forms during next fetch cycle). If false, form action attribute will be ignored. + + + parser.rss.aggregate_entries + true + If true, the RSS parser will return a single Parse + containing an aggregation of all feed entries. If false, it will + create a Parse for each entry. + - Index: src/java/org/apache/nutch/fetcher/Fetcher.java =================================================================== --- src/java/org/apache/nutch/fetcher/Fetcher.java (revision 505236) +++ src/java/org/apache/nutch/fetcher/Fetcher.java (working copy) @@ -18,6 +18,9 @@ package org.apache.nutch.fetcher; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; // Commons Logging imports import org.apache.commons.logging.Log; @@ -283,7 +286,7 @@ datum.setStatus(status); datum.setFetchTime(System.currentTimeMillis()); if (pstatus != null) datum.getMetaData().put(Nutch.WRITABLE_PROTO_STATUS_KEY, pstatus); - + if (content == null) { String url = key.toString(); content = new Content(url, url, new byte[0], "", new Metadata(), this.conf); @@ -301,54 +304,79 @@ } } - Parse parse = null; + Map parseMap = null; if (parsing && status == CrawlDatum.STATUS_FETCH_SUCCESS) { - ParseStatus parseStatus; try { - parse = this.parseUtil.parse(content); - parseStatus = parse.getData().getStatus(); + parseMap = this.parseUtil.parse(content); } catch (Exception e) { - parseStatus = new ParseStatus(e); + parseMap = new ParseStatus(e).getEmptyParseMap(conf); } - if (!parseStatus.isSuccess()) { - if (LOG.isWarnEnabled()) { - LOG.warn("Error parsing: " + key + ": " + parseStatus); - } - parse = parseStatus.getEmptyParse(getConf()); - } + // Calculate page signature. For non-parsing fetchers this will // be done in ParseSegment - byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, parse); + byte[] signature = + SignatureFactory.getSignature(getConf()).calculate(content, parseMap); metadata.set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature)); datum.setSignature(signature); - // Ensure segment name and score are in parseData metadata - parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName); - parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature)); - try { - scfilters.passScoreAfterParsing(key, content, parse); - } catch (Exception e) { - if (LOG.isWarnEnabled()) { - e.printStackTrace(LogUtil.getWarnStream(LOG)); - LOG.warn("Couldn't pass score, url " + key + " (" + e + ")"); + + for (Entry entry : parseMap.entrySet()) { + Text url = new Text(entry.getKey()); + Parse parse = entry.getValue(); + ParseStatus parseStatus = parse.getData().getStatus(); + Map singleEntryMap = new HashMap(); + + if (!parseStatus.isSuccess()) { + if (LOG.isWarnEnabled()) { + LOG.warn("Error parsing: " + key + ": " + parseStatus); + } + parse = parseStatus.getEmptyParse(getConf()); } + + // Calculate signature + singleEntryMap.put(entry.getKey(), parse); + byte[] entrySig = + SignatureFactory.getSignature(getConf()).calculate(content, singleEntryMap); + // Ensure segment name and score are in parseData metadata + parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName); + parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY, + StringUtil.toHexString(entrySig)); + try { + scfilters.passScoreAfterParsing(url, content, parse); + } catch (Exception e) { + if (LOG.isWarnEnabled()) { + e.printStackTrace(LogUtil.getWarnStream(LOG)); + LOG.warn("Couldn't pass score, url " + key + " (" + e + ")"); + } + } } } try { - output.collect - (key, - new FetcherOutput(datum, - storingContent ? content : null, - parse != null ? new ParseImpl(parse) : null)); + output.collect(key, new ObjectWritable(datum)); + if (storingContent) + output.collect(key, new ObjectWritable(content)); + if (parseMap != null) { + for (Entry entry : parseMap.entrySet()) { + output.collect(new Text(entry.getKey()), + new ObjectWritable(new ParseImpl(entry.getValue()))); + } + } } catch (IOException e) { if (LOG.isFatalEnabled()) { e.printStackTrace(LogUtil.getFatalStream(LOG)); LOG.fatal("fetcher caught:"+e.toString()); } } - if (parse != null) return parse.getData().getStatus(); - else return null; + + // return first parse status if it exits + if (parseMap != null && !parseMap.isEmpty()) { + ParseData pd = parseMap.values().iterator().next().getData(); + if (pd != null) { + return pd.getStatus(); + } + } + return null; } } @@ -465,7 +493,7 @@ job.setOutputPath(segment); job.setOutputFormat(FetcherOutputFormat.class); job.setOutputKeyClass(Text.class); - job.setOutputValueClass(FetcherOutput.class); + job.setOutputValueClass(ObjectWritable.class); JobClient.runJob(job); if (LOG.isInfoEnabled()) { LOG.info("Fetcher: done"); } Index: src/java/org/apache/nutch/fetcher/FetcherOutput.java =================================================================== --- src/java/org/apache/nutch/fetcher/FetcherOutput.java (revision 505236) +++ src/java/org/apache/nutch/fetcher/FetcherOutput.java (working copy) @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nutch.fetcher; - -import java.io.*; - -import org.apache.hadoop.io.*; -import org.apache.nutch.crawl.CrawlDatum; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.parse.*; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configurable; - -/* An entry in the fetcher's output. */ -public final class FetcherOutput implements Writable, Configurable { - private CrawlDatum crawlDatum; - private Content content; - private ParseImpl parse; - private Configuration conf; - - public FetcherOutput() {} - - public FetcherOutput(CrawlDatum crawlDatum, Content content, - ParseImpl parse) { - this.crawlDatum = crawlDatum; - this.content = content; - this.parse = parse; - } - - public final void readFields(DataInput in) throws IOException { - this.crawlDatum = CrawlDatum.read(in); - this.content = in.readBoolean() ? Content.read(in) : null; - this.parse = in.readBoolean() ? ParseImpl.read(in, this.conf) : null; - } - - public final void write(DataOutput out) throws IOException { - crawlDatum.write(out); - - out.writeBoolean(content != null); - if (content != null) { - content.write(out); - } - - out.writeBoolean(parse != null); - if (parse != null) { - parse.write(out); - } - } - - public CrawlDatum getCrawlDatum() { return crawlDatum; } - public Content getContent() { return content; } - public ParseImpl getParse() { return parse; } - - public boolean equals(Object o) { - if (!(o instanceof FetcherOutput)) - return false; - FetcherOutput other = (FetcherOutput)o; - return - this.crawlDatum.equals(other.crawlDatum) && - this.content.equals(other.content); - } - - public String toString() { - StringBuffer buffer = new StringBuffer(); - buffer.append("CrawlDatum: " + crawlDatum+"\n" ); - return buffer.toString(); - } - - public void setConf(Configuration conf) { - this.conf = conf; - } - - public Configuration getConf() { - return this.conf; - } - -} Index: src/java/org/apache/nutch/fetcher/Fetcher2.java =================================================================== --- src/java/org/apache/nutch/fetcher/Fetcher2.java (revision 505236) +++ src/java/org/apache/nutch/fetcher/Fetcher2.java (working copy) @@ -21,6 +21,7 @@ import java.net.URL; import java.net.UnknownHostException; import java.util.*; +import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -663,54 +664,78 @@ } } - Parse parse = null; + Map parseMap = null; if (parsing && status == CrawlDatum.STATUS_FETCH_SUCCESS) { - ParseStatus parseStatus; try { - parse = this.parseUtil.parse(content); - parseStatus = parse.getData().getStatus(); + parseMap = this.parseUtil.parse(content); } catch (Exception e) { - parseStatus = new ParseStatus(e); + // ignore } - if (!parseStatus.isSuccess()) { - if (LOG.isWarnEnabled()) { - LOG.warn("Error parsing: " + key + ": " + parseStatus); - } - parse = parseStatus.getEmptyParse(getConf()); - } + // Calculate page signature. For non-parsing fetchers this will // be done in ParseSegment - byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, parse); + byte[] signature = + SignatureFactory.getSignature(getConf()).calculate(content, parseMap); metadata.set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature)); datum.setSignature(signature); - // Ensure segment name and score are in parseData metadata - parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName); - parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature)); - try { - scfilters.passScoreAfterParsing(key, content, parse); - } catch (Exception e) { - if (LOG.isWarnEnabled()) { - e.printStackTrace(LogUtil.getWarnStream(LOG)); - LOG.warn("Couldn't pass score, url " + key + " (" + e + ")"); + + for (Entry entry : parseMap.entrySet()) { + Text url = new Text(entry.getKey()); + Parse parse = entry.getValue(); + ParseStatus parseStatus = parse.getData().getStatus(); + Map singleEntryMap = new HashMap(); + + if (!parseStatus.isSuccess()) { + if (LOG.isWarnEnabled()) { + LOG.warn("Error parsing: " + key + ": " + parseStatus); + } + parse = parseStatus.getEmptyParse(getConf()); } + + // Calculate signature + singleEntryMap.put(entry.getKey(), parse); + byte[] entrySig = + SignatureFactory.getSignature(getConf()).calculate(content, singleEntryMap); + // Ensure segment name and score are in parseData metadata + parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName); + parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY, + StringUtil.toHexString(entrySig)); + try { + scfilters.passScoreAfterParsing(url, content, parse); + } catch (Exception e) { + if (LOG.isWarnEnabled()) { + e.printStackTrace(LogUtil.getWarnStream(LOG)); + LOG.warn("Couldn't pass score, url " + key + " (" + e + ")"); + } + } } } try { - output.collect - (key, - new FetcherOutput(datum, - storingContent ? content : null, - parse != null ? new ParseImpl(parse) : null)); + output.collect(key, new ObjectWritable(datum)); + if (storingContent) + output.collect(key, new ObjectWritable(content)); + if (parseMap != null) { + for (Entry entry : parseMap.entrySet()) { + output.collect(new Text(entry.getKey()), + new ObjectWritable(new ParseImpl(entry.getValue()))); + } + } } catch (IOException e) { if (LOG.isFatalEnabled()) { e.printStackTrace(LogUtil.getFatalStream(LOG)); LOG.fatal("fetcher caught:"+e.toString()); } } - if (parse != null) return parse.getData().getStatus(); - else return null; + + if (parseMap != null && !parseMap.isEmpty()) { + ParseData pd = parseMap.values().iterator().next().getData(); + if (pd != null) { + return pd.getStatus(); + } + } + return null; } } @@ -833,7 +858,7 @@ job.setOutputPath(segment); job.setOutputFormat(FetcherOutputFormat.class); job.setOutputKeyClass(Text.class); - job.setOutputValueClass(FetcherOutput.class); + job.setOutputValueClass(ObjectWritable.class); JobClient.runJob(job); if (LOG.isInfoEnabled()) { LOG.info("Fetcher: done"); } Index: src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java =================================================================== --- src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (revision 505236) +++ src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (working copy) @@ -18,12 +18,15 @@ package org.apache.nutch.fetcher; import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; import org.apache.nutch.crawl.CrawlDatum; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MapFile; +import org.apache.hadoop.io.ObjectWritable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.Text; @@ -34,6 +37,8 @@ import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.Progressable; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseImpl; import org.apache.nutch.parse.ParseOutputFormat; import org.apache.nutch.protocol.Content; @@ -76,18 +81,15 @@ public void write(WritableComparable key, Writable value) throws IOException { - FetcherOutput fo = (FetcherOutput)value; + ObjectWritable ow = (ObjectWritable)value; + Writable w = (Writable) ow.get(); - fetchOut.append(key, fo.getCrawlDatum()); - - if (fo.getContent() != null) { - contentOut.append(key, fo.getContent()); - } - - if (fo.getParse() != null) { - parseOut.write(key, fo.getParse()); - } - + if (w instanceof CrawlDatum) + fetchOut.append(key, w); + else if (w instanceof Content) + contentOut.append(key, w); + else if (w instanceof Parse) + parseOut.write(key, w); } public void close(Reporter reporter) throws IOException { Index: src/java/org/apache/nutch/crawl/Signature.java =================================================================== --- src/java/org/apache/nutch/crawl/Signature.java (revision 505236) +++ src/java/org/apache/nutch/crawl/Signature.java (working copy) @@ -17,6 +17,8 @@ package org.apache.nutch.crawl; +import java.util.Map; + import org.apache.nutch.parse.Parse; import org.apache.nutch.protocol.Content; import org.apache.hadoop.conf.Configuration; @@ -25,7 +27,7 @@ public abstract class Signature implements Configurable { protected Configuration conf; - public abstract byte[] calculate(Content content, Parse parse); + public abstract byte[] calculate(Content content, Map parseMap); public Configuration getConf() { return conf; Index: src/java/org/apache/nutch/crawl/MapWritable.java =================================================================== --- src/java/org/apache/nutch/crawl/MapWritable.java (revision 505236) +++ src/java/org/apache/nutch/crawl/MapWritable.java (working copy) @@ -85,17 +85,15 @@ addToMap(LongWritable.class, new Byte((byte) -126)); addToMap(Text.class, new Byte((byte) -125)); addToMap(MD5Hash.class, new Byte((byte) -124)); - addToMap(org.apache.nutch.fetcher.FetcherOutput.class, - new Byte((byte) -123)); - addToMap(org.apache.nutch.protocol.Content.class, new Byte((byte) -122)); - addToMap(org.apache.nutch.parse.ParseText.class, new Byte((byte) -121)); - addToMap(org.apache.nutch.parse.ParseData.class, new Byte((byte) -120)); - addToMap(MapWritable.class, new Byte((byte) -119)); - addToMap(BytesWritable.class, new Byte((byte) -118)); - addToMap(FloatWritable.class, new Byte((byte) -117)); - addToMap(IntWritable.class, new Byte((byte) -116)); - addToMap(ObjectWritable.class, new Byte((byte) -115)); - addToMap(ProtocolStatus.class, new Byte((byte) -114)); + addToMap(org.apache.nutch.protocol.Content.class, new Byte((byte) -123)); + addToMap(org.apache.nutch.parse.ParseText.class, new Byte((byte) -122)); + addToMap(org.apache.nutch.parse.ParseData.class, new Byte((byte) -121)); + addToMap(MapWritable.class, new Byte((byte) -120)); + addToMap(BytesWritable.class, new Byte((byte) -119)); + addToMap(FloatWritable.class, new Byte((byte) -118)); + addToMap(IntWritable.class, new Byte((byte) -117)); + addToMap(ObjectWritable.class, new Byte((byte) -116)); + addToMap(ProtocolStatus.class, new Byte((byte) -115)); } Index: src/java/org/apache/nutch/crawl/MD5Signature.java =================================================================== --- src/java/org/apache/nutch/crawl/MD5Signature.java (revision 505236) +++ src/java/org/apache/nutch/crawl/MD5Signature.java (working copy) @@ -17,6 +17,8 @@ package org.apache.nutch.crawl; +import java.util.Map; + import org.apache.hadoop.io.MD5Hash; import org.apache.nutch.parse.Parse; import org.apache.nutch.protocol.Content; @@ -30,7 +32,7 @@ */ public class MD5Signature extends Signature { - public byte[] calculate(Content content, Parse parse) { + public byte[] calculate(Content content, Map parseMap) { byte[] data = content.getContent(); if (data == null) data = content.getUrl().getBytes(); return MD5Hash.digest(data).getDigest(); Index: src/java/org/apache/nutch/crawl/TextProfileSignature.java =================================================================== --- src/java/org/apache/nutch/crawl/TextProfileSignature.java (revision 505236) +++ src/java/org/apache/nutch/crawl/TextProfileSignature.java (working copy) @@ -26,6 +26,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import org.apache.hadoop.io.MD5Hash; import org.apache.nutch.parse.Parse; @@ -63,13 +64,19 @@ Signature fallback = new MD5Signature(); - public byte[] calculate(Content content, Parse parse) { + public byte[] calculate(Content content, Map parseMap) { int MIN_TOKEN_LEN = getConf().getInt("db.signature.text_profile.min_token_len", 2); float QUANT_RATE = getConf().getFloat("db.signature.text_profile.quant_rate", 0.01f); HashMap tokens = new HashMap(); String text = null; - if (parse != null) text = parse.getText(); - if (text == null || text.length() == 0) return fallback.calculate(content, parse); + if (parseMap != null) { + StringBuffer textBuf = new StringBuffer(); + for (Parse parse : parseMap.values()) { + textBuf.append(parse.getText()); + } + text = textBuf.toString(); + } + if (text == null || text.length() == 0) return fallback.calculate(content, parseMap); StringBuffer curToken = new StringBuffer(); int maxFreq = 0; for (int i = 0; i < text.length(); i++) { @@ -171,7 +178,9 @@ text.append(line); } br.close(); - byte[] signature = sig.calculate(null, new ParseImpl(text.toString(), null)); + Map parseMap = new HashMap(); + parseMap.put("", new ParseImpl(text.toString(), null)); + byte[] signature = sig.calculate(null, parseMap); res.put(files[i].toString(), signature); } Iterator it = res.keySet().iterator(); Index: src/java/org/apache/nutch/parse/ParseUtil.java =================================================================== --- src/java/org/apache/nutch/parse/ParseUtil.java (revision 505236) +++ src/java/org/apache/nutch/parse/ParseUtil.java (working copy) @@ -17,6 +17,9 @@ package org.apache.nutch.parse; // Commons Logging imports +import java.util.HashMap; +import java.util.Map; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -59,10 +62,10 @@ * WARNING level, and an empty parse is returned. * * @param content The content to try and parse. - * @return A {@link Parse} object containing the parsed data. + * @return <key, {@link Parse}> pairs. * @throws ParseException If no suitable parser is found to perform the parse. */ - public Parse parse(Content content) throws ParseException { + public Map parse(Content content) throws ParseException { Parser[] parsers = null; try { @@ -76,25 +79,21 @@ throw new ParseException(e.getMessage()); } - Parse parse = null; + Map parseMap = null; for (int i=0; iextId, i.e., the Parser's extension ID. * If a suitable {@link Parser} is not found, then a WARNING * level message is logged, and a ParseException is thrown. If the parse is - * uncessful for any other reason, then a WARNING level + * unsuccessful for any other reason, then a WARNING level * message is logged, and a ParseStatus.getEmptyParse() is * returned. * @@ -110,15 +109,14 @@ * to parse the specified content. * @param content The content to parse. * - * @return A {@link Parse} object if the parse is successful, otherwise, - * a ParseStatus.getEmptyParse(). + * @return <key, {@link Parse}> pairs if the parse is successful, otherwise, + * a single <key, ParseStatus.getEmptyParse()> pair. * * @throws ParseException If there is no suitable {@link Parser} found * to perform the parse. */ - public Parse parseByExtensionId(String extId, Content content) + public Map parseByExtensionId(String extId, Content content) throws ParseException { - Parse parse = null; Parser p = null; try { @@ -131,17 +129,29 @@ throw new ParseException(e.getMessage()); } - parse = p.getParse(content); - - if (parse != null && parse.getData().getStatus().isSuccess()) { - return parse; + Map parseMap = p.getParse(content); + if (processParseMap(content, parseMap)) { + return parseMap; } else { - if (LOG.isWarnEnabled()) { + if (LOG.isWarnEnabled()) { LOG.warn("Unable to successfully parse content " + content.getUrl() + - " of type " + content.getContentType()); + " of type " + content.getContentType()); } - return new ParseStatus().getEmptyParse(this.conf); + return new ParseStatus().getEmptyParseMap(conf); } - } + } + private boolean processParseMap(Content content, + Map parseMap) { + if (parseMap != null) { + for (String key : parseMap.keySet()) { + Parse parse = parseMap.get(key); + if (parse == null || !parse.getData().getStatus().isSuccess()) + parseMap.remove(key); + } + return !parseMap.isEmpty(); + } + + return false; + } } Index: src/java/org/apache/nutch/parse/ParseStatus.java =================================================================== --- src/java/org/apache/nutch/parse/ParseStatus.java (revision 505236) +++ src/java/org/apache/nutch/parse/ParseStatus.java (working copy) @@ -24,6 +24,8 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Map; +import java.util.HashMap; import org.apache.hadoop.io.VersionedWritable; import org.apache.hadoop.io.WritableUtils; @@ -184,7 +186,16 @@ public Parse getEmptyParse(Configuration conf) { return new EmptyParseImpl(this, conf); } - + + /** A convenience method. Creates a Map with a single + * empty Parse instance, which returns this status. + */ + public Map getEmptyParseMap(Configuration conf) { + Map parseMap = new HashMap(); + parseMap.put("", new EmptyParseImpl(this, conf)); + return parseMap; + } + public String toString() { StringBuffer res = new StringBuffer(); String name = null; Index: src/java/org/apache/nutch/parse/ParserChecker.java =================================================================== --- src/java/org/apache/nutch/parse/ParserChecker.java (revision 505236) +++ src/java/org/apache/nutch/parse/ParserChecker.java (working copy) @@ -17,6 +17,9 @@ package org.apache.nutch.parse; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -94,13 +97,18 @@ LOG.info("contentType: "+contentType); } - Parse parse = new ParseUtil(conf).parse(content); + Map parseMap = new ParseUtil(conf).parse(content); - System.out.print("---------\nParseData\n---------\n"); - System.out.print(parse.getData().toString()); - if (dumpText) { - System.out.print("---------\nParseText\n---------\n"); - System.out.print(parse.getText()); + for (Entry entry : parseMap.entrySet()) { + Parse parse = entry.getValue(); + System.out.print("---------\nUrl\n---------------\n"); + System.out.print(entry.getKey()); + System.out.print("---------\nParseData\n---------\n"); + System.out.print(parse.getData().toString()); + if (dumpText) { + System.out.print("---------\nParseText\n---------\n"); + System.out.print(parse.getText()); + } } System.exit(0); Index: src/java/org/apache/nutch/parse/HtmlParseFilters.java =================================================================== --- src/java/org/apache/nutch/parse/HtmlParseFilters.java (revision 505236) +++ src/java/org/apache/nutch/parse/HtmlParseFilters.java (working copy) @@ -17,6 +17,7 @@ package org.apache.nutch.parse; +import java.util.Map; import java.util.HashMap; import org.apache.nutch.protocol.Content; @@ -56,13 +57,19 @@ } /** Run all defined filters. */ - public Parse filter(Content content, Parse parse, HTMLMetaTags metaTags, DocumentFragment doc) { + public Map filter(Content content, Map parseMap, HTMLMetaTags metaTags, DocumentFragment doc) { - for (int i = 0 ; i < this.htmlParseFilters.length; i++) { - parse = this.htmlParseFilters[i].filter(content, parse, metaTags, doc); - if (!parse.getData().getStatus().isSuccess()) break; + Map filteredParseMap = new HashMap(); + + for (String key : parseMap.keySet() ) { + Parse parse = parseMap.get(key); + for (int i = 0 ; i < this.htmlParseFilters.length; i++) { + parse = this.htmlParseFilters[i].filter(content, parse, metaTags, doc); + if (!parse.getData().getStatus().isSuccess()) break; + } + filteredParseMap.put(key, parse); } - return parse; + return filteredParseMap; } } Index: src/java/org/apache/nutch/parse/ParseImpl.java =================================================================== --- src/java/org/apache/nutch/parse/ParseImpl.java (revision 505236) +++ src/java/org/apache/nutch/parse/ParseImpl.java (working copy) @@ -18,6 +18,7 @@ package org.apache.nutch.parse; import java.io.*; +import java.util.*; import org.apache.hadoop.io.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configurable; @@ -46,6 +47,36 @@ this.data = data; } + /** + * convenience method + * @return a Map containing a single entry + */ + public static Map createSingleEntryMap(String text, ParseData data, String key) { + return createSingleEntryMap(new ParseText(text), data, key); + } + + /** + * convenience method + * @return a Map containing a single entry + */ + public static Map createSingleEntryMap(ParseText text, ParseData data, String key) { + Map parseMap = new HashMap(); + parseMap.put(key, new ParseImpl(text, data)); + return parseMap; + } + + /** + * convenience method + * @return the first Parse object of this Map + */ + public static Parse getFirstParseEntry(Map parseMap) { + + if (parseMap != null && parseMap.size() > 0) { + return parseMap.values().iterator().next(); + } // else + return null; + } + public String getText() { return text.getText(); } public ParseData getData() { return data; } Index: src/java/org/apache/nutch/parse/Parser.java =================================================================== --- src/java/org/apache/nutch/parse/Parser.java (revision 505236) +++ src/java/org/apache/nutch/parse/Parser.java (working copy) @@ -18,6 +18,8 @@ package org.apache.nutch.parse; // Hadoop imports +import java.util.Map; + import org.apache.hadoop.conf.Configurable; // Nutch imports @@ -33,5 +35,5 @@ public final static String X_POINT_ID = Parser.class.getName(); /** Creates the parse for some content. */ - Parse getParse(Content c); + Map getParse(Content c); } Index: src/java/org/apache/nutch/parse/ParseSegment.java =================================================================== --- src/java/org/apache/nutch/parse/ParseSegment.java (revision 505236) +++ src/java/org/apache/nutch/parse/ParseSegment.java (working copy) @@ -34,6 +34,7 @@ import java.io.*; import java.util.*; +import java.util.Map.Entry; /* Parse content in a segment. */ public class ParseSegment extends Configured implements Mapper, Reducer { @@ -70,32 +71,35 @@ Content content = (Content) value; content.forceInflate(); - Parse parse = null; - ParseStatus status; + Map parseMap = null; try { - parse = new ParseUtil(getConf()).parse(content); - status = parse.getData().getStatus(); + parseMap = new ParseUtil(getConf()).parse(content); } catch (Exception e) { - status = new ParseStatus(e); + // ignore } - - // compute the new signature - byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, parse); + + byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, parseMap); content.getMetadata().set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature)); - if (status.isSuccess()) { - try { - scfilters.passScoreAfterParsing((Text)key, content, parse); - } catch (ScoringFilterException e) { - if (LOG.isWarnEnabled()) { - e.printStackTrace(LogUtil.getWarnStream(LOG)); - LOG.warn("Error passing score: "+key+": "+e.getMessage()); + for (Entry entry : parseMap.entrySet()) { + Text url = new Text(entry.getKey()); + Parse parse = entry.getValue(); + ParseStatus status = parse.getData().getStatus(); + // compute the new signature + if (status.isSuccess()) { + try { + scfilters.passScoreAfterParsing(url, content, parse); + } catch (ScoringFilterException e) { + if (LOG.isWarnEnabled()) { + e.printStackTrace(LogUtil.getWarnStream(LOG)); + LOG.warn("Error passing score: "+ url +": "+e.getMessage()); + } + return; } - return; + output.collect(url, new ParseImpl(parse.getText(), parse.getData())); + } else if (LOG.isWarnEnabled()) { + LOG.warn("Error parsing: " + url + ": "+status.toString()); } - output.collect(key, new ParseImpl(parse.getText(), parse.getData())); - } else if (LOG.isWarnEnabled()) { - LOG.warn("Error parsing: "+key+": "+status.toString()); } } Index: src/java/org/apache/nutch/util/NutchConfiguration.java =================================================================== --- src/java/org/apache/nutch/util/NutchConfiguration.java (revision 505236) +++ src/java/org/apache/nutch/util/NutchConfiguration.java (working copy) @@ -39,8 +39,6 @@ // for back-compatibility, add old aliases for these Writable classes // this may be removed after the 0.8 release static { - WritableName.addName(org.apache.nutch.fetcher.FetcherOutput.class, - "FetcherOutput"); WritableName.addName(org.apache.nutch.parse.ParseData.class, "ParseData"); WritableName.addName(org.apache.nutch.parse.ParseText.class, "ParseText"); WritableName.addName(org.apache.nutch.protocol.Content.class, "Content"); Index: src/plugin/parse-msword/src/java/org/apache/nutch/parse/msword/MSWordParser.java =================================================================== --- src/plugin/parse-msword/src/java/org/apache/nutch/parse/msword/MSWordParser.java (revision 505236) +++ src/plugin/parse-msword/src/java/org/apache/nutch/parse/msword/MSWordParser.java (working copy) @@ -16,10 +16,12 @@ */ package org.apache.nutch.parse.msword; +import java.util.Map; + // Nutch imports -import org.apache.nutch.protocol.Content; import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ms.MSBaseParser; +import org.apache.nutch.protocol.Content; /** @@ -40,7 +42,7 @@ public static final String MIME_TYPE = "application/msword"; - public Parse getParse(Content content) { + public Map getParse(Content content) { return getParse(new WordExtractor(), content); } Index: src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java =================================================================== --- src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java (revision 505236) +++ src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java (working copy) @@ -25,10 +25,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - +import org.apache.hadoop.conf.Configuration; import org.apache.nutch.parse.HTMLMetaTags; import org.apache.nutch.parse.HtmlParseFilter; import org.apache.nutch.parse.Outlink; @@ -39,7 +40,6 @@ import org.apache.nutch.parse.Parser; import org.apache.nutch.protocol.Content; import org.apache.nutch.util.NutchConfiguration; -import org.apache.hadoop.conf.Configuration; import org.apache.oro.text.regex.MatchResult; import org.apache.oro.text.regex.Pattern; import org.apache.oro.text.regex.PatternCompiler; @@ -141,11 +141,11 @@ } } - public Parse getParse(Content c) { + public Map getParse(Content c) { String type = c.getContentType(); if (type != null && !type.trim().equals("") && !type.toLowerCase().startsWith("application/x-javascript")) return new ParseStatus(ParseStatus.FAILED_INVALID_FORMAT, - "Content not JavaScript: '" + type + "'").getEmptyParse(getConf()); + "Content not JavaScript: '" + type + "'").getEmptyParseMap(getConf()); String script = new String(c.getContent()); Outlink[] outlinks = getJSLinks(script, "", c.getUrl()); if (outlinks == null) outlinks = new Outlink[0]; @@ -162,8 +162,7 @@ ParseData pd = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, c.getMetadata()); pd.setConf(this.conf); - Parse parse = new ParseImpl(script, pd); - return parse; + return ParseImpl.createSingleEntryMap(script, pd, c.getUrl()); } private static final String STRING_PATTERN = "(\\\\*(?:\"|\'))([^\\s\"\']+?)(?:\\1)"; Index: src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java =================================================================== --- src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java (revision 505236) +++ src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java (working copy) @@ -45,6 +45,7 @@ import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseUtil; +import org.apache.nutch.parse.ParseImpl; import org.apache.nutch.parse.ParseException; import org.apache.nutch.parse.ParserNotFound; import org.apache.nutch.protocol.Content; @@ -346,7 +347,8 @@ try { protocol = new ProtocolFactory(conf).getProtocol(url); Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent(); - Parse parse = new ParseUtil(conf).parse(content); + Parse parse = + ParseImpl.getFirstParseEntry(new ParseUtil(conf).parse(content)); System.out.println("text:" + parse.getText()); return parse.getText(); Index: src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java =================================================================== --- src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java (revision 505236) +++ src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java (working copy) @@ -61,7 +61,7 @@ return conf; } - public Parse getParse(Content content) { + public Map getParse(Content content) { String text = null; Vector outlinks = new Vector(); @@ -74,7 +74,7 @@ if (contentLength != null && raw.length != Integer.parseInt(contentLength)) { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_TRUNCATED, "Content truncated at " + raw.length + - " bytes. Parser can't handle incomplete files.").getEmptyParse(conf); + " bytes. Parser can't handle incomplete files.").getEmptyParseMap(conf); } ExtractText extractor = new ExtractText(); @@ -103,14 +103,14 @@ } } catch (Exception e) { // run time exception e.printStackTrace(LogUtil.getErrorStream(LOG)); - return new ParseStatus(ParseStatus.FAILED, "Can't be handled as SWF document. " + e).getEmptyParse(conf); - } finally {} + return new ParseStatus(ParseStatus.FAILED, "Can't be handled as SWF document. " + e).getEmptyParseMap(conf); + } if (text == null) text = ""; Outlink[] links = (Outlink[]) outlinks.toArray(new Outlink[outlinks.size()]); ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, "", links, content.getMetadata()); - return new ParseImpl(text, parseData); + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); } /** @@ -122,10 +122,11 @@ byte[] buf = new byte[in.available()]; in.read(buf); SWFParser parser = new SWFParser(); - Parse p = parser.getParse(new Content("file:" + args[0], "file:" + args[0], + Map parseMap = parser.getParse(new Content("file:" + args[0], "file:" + args[0], buf, "application/x-shockwave-flash", new Metadata(), NutchConfiguration.create())); + Parse p = ParseImpl.getFirstParseEntry(parseMap); System.out.println("Parse Text:"); System.out.println(p.getText()); System.out.println("Parse Data:"); Index: src/plugin/parse-msexcel/src/java/org/apache/nutch/parse/msexcel/MSExcelParser.java =================================================================== --- src/plugin/parse-msexcel/src/java/org/apache/nutch/parse/msexcel/MSExcelParser.java (revision 505236) +++ src/plugin/parse-msexcel/src/java/org/apache/nutch/parse/msexcel/MSExcelParser.java (working copy) @@ -16,6 +16,8 @@ */ package org.apache.nutch.parse.msexcel; +import java.util.Map; + // Nutch imports import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ms.MSBaseParser; @@ -37,7 +39,7 @@ public static final String MIME_TYPE = "application/vnd.ms-excel"; - public Parse getParse(Content content) { + public Map getParse(Content content) { return getParse(new ExcelExtractor(), content); } Index: src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java =================================================================== --- src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java (revision 505236) +++ src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java (working copy) @@ -19,14 +19,14 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.Properties; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - -import org.apache.nutch.metadata.Metadata; +import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.parse.Outlink; import org.apache.nutch.parse.Parse; @@ -35,7 +35,6 @@ import org.apache.nutch.parse.ParseStatus; import org.apache.nutch.parse.Parser; import org.apache.nutch.protocol.Content; -import org.apache.hadoop.conf.Configuration; /** * ZipParser class based on MSPowerPointParser class by Stephan Strittmatter. @@ -52,7 +51,7 @@ public ZipParser() { } - public Parse getParse(final Content content) { + public Map getParse(final Content content) { String resultText = null; String resultTitle = null; @@ -74,7 +73,7 @@ ParseStatus.FAILED_TRUNCATED, "Content truncated at " + contentInBytes.length + " bytes. Parser can't handle incomplete pdf file.") - .getEmptyParse(getConf()); + .getEmptyParseMap(getConf()); } ZipTextExtractor extractor = new ZipTextExtractor(getConf()); @@ -85,7 +84,7 @@ } catch (Exception e) { return new ParseStatus(ParseStatus.FAILED, - "Can't be handled as Zip document. " + e).getEmptyParse(getConf()); + "Can't be handled as Zip document. " + e).getEmptyParseMap(getConf()); } if (resultText == null) { @@ -103,7 +102,7 @@ parseData.setConf(this.conf); if (LOG.isTraceEnabled()) { LOG.trace("Zip file parsed sucessfully !!"); } - return new ParseImpl(resultText, parseData); + return ParseImpl.createSingleEntryMap(resultText, parseData, content.getUrl()); } public void setConf(Configuration conf) { Index: src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java =================================================================== --- src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java (revision 505236) +++ src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java (working copy) @@ -38,6 +38,7 @@ import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseData; import org.apache.nutch.parse.ParseUtil; +import org.apache.nutch.parse.ParseImpl; import org.apache.nutch.parse.ParseException; import org.apache.nutch.parse.Outlink; import org.apache.nutch.protocol.Content; @@ -97,7 +98,7 @@ metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize())); metadata.set(Response.CONTENT_TYPE, contentType); Content content = new Content(newurl, base, b, contentType, metadata, this.conf); - Parse parse = new ParseUtil(this.conf).parse(content); + Parse parse = ParseImpl.getFirstParseEntry(new ParseUtil(this.conf).parse(content)); ParseData theParseData = parse.getData(); Outlink[] theOutlinks = theParseData.getOutlinks(); Index: src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java =================================================================== --- src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java (revision 505236) +++ src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java (working copy) @@ -88,7 +88,8 @@ protocol = new ProtocolFactory(conf).getProtocol(urlString); content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); - parse = new ParseUtil(conf).parseByExtensionId("parse-rss", content); + parse = ParseImpl.getFirstParseEntry(new ParseUtil(conf) + .parseByExtensionId("parse-rss", content)); //check that there are 3 outlinks: //http://test.channel.com Index: src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java =================================================================== --- src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java (revision 505236) +++ src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java (working copy) @@ -22,6 +22,7 @@ import java.net.MalformedURLException; import java.util.List; import java.util.Vector; +import java.util.Map; // Commons Logging imports import org.apache.commons.logging.Log; @@ -76,7 +77,7 @@ * The content to parse (hopefully an RSS content stream) * @return A {@link ParseImpl}which implements the {@link Parse}interface. */ - public Parse getParse(Content content) { + public Map getParse(Content content) { List theRSSChannels = null; @@ -101,7 +102,8 @@ LOG.warn("nutch:parse-rss:RSSParser Exception: " + e.getMessage()); } return new ParseStatus(ParseStatus.FAILED, - "Can't be handled as rss document. " + e).getEmptyParse(getConf()); + "Can't be handled as rss document. " + e) + .getEmptyParseMap(getConf()); } StringBuffer contentTitle = new StringBuffer(), indexText = new StringBuffer(); @@ -199,7 +201,8 @@ ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, contentTitle.toString(), outlinks, content.getMetadata()); parseData.setConf(this.conf); - return new ParseImpl(indexText.toString(), parseData); + return ParseImpl.createSingleEntryMap(indexText.toString(), parseData, + content.getUrl()); } public void setConf(Configuration conf) { @@ -218,7 +221,7 @@ parser.setConf(conf); Protocol protocol = new ProtocolFactory(conf).getProtocol(url); Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent(); - Parse parse = parser.getParse(content); + Parse parse = ParseImpl.getFirstParseEntry(parser.getParse(content)); System.out.println("data: "+ parse.getData()); System.out.println("text: "+parse.getText()); } Index: src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java =================================================================== --- src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java (revision 505236) +++ src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java (working copy) @@ -17,38 +17,34 @@ package org.apache.nutch.parse.pdf; -import org.pdfbox.encryption.DocumentEncryption; -import org.pdfbox.pdfparser.PDFParser; -import org.pdfbox.pdmodel.PDDocument; -import org.pdfbox.pdmodel.PDDocumentInformation; -import org.pdfbox.util.PDFTextStripper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Map; -import org.pdfbox.exceptions.CryptographyException; -import org.pdfbox.exceptions.InvalidPasswordException; - -// Commons Logging imports import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - -import org.apache.nutch.protocol.Content; +import org.apache.hadoop.conf.Configuration; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.net.protocols.Response; -import org.apache.hadoop.conf.Configuration; -import org.apache.nutch.parse.ParseStatus; -import org.apache.nutch.parse.Parser; +import org.apache.nutch.parse.Outlink; +import org.apache.nutch.parse.OutlinkExtractor; import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseData; import org.apache.nutch.parse.ParseImpl; -import org.apache.nutch.parse.Outlink; -import org.apache.nutch.parse.OutlinkExtractor; +import org.apache.nutch.parse.ParseStatus; +import org.apache.nutch.parse.Parser; +import org.apache.nutch.protocol.Content; import org.apache.nutch.util.LogUtil; +import org.pdfbox.encryption.DocumentEncryption; +import org.pdfbox.exceptions.CryptographyException; +import org.pdfbox.exceptions.InvalidPasswordException; +import org.pdfbox.pdfparser.PDFParser; +import org.pdfbox.pdmodel.PDDocument; +import org.pdfbox.pdmodel.PDDocumentInformation; +import org.pdfbox.util.PDFTextStripper; -import java.text.SimpleDateFormat; -import java.util.Calendar; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - /********************************************* * parser for mime type application/pdf. * It is based on org.pdfbox.*. We have to see how well it does the job. @@ -66,7 +62,7 @@ public static final Log LOG = LogFactory.getLog("org.apache.nutch.parse.pdf"); private Configuration conf; - public Parse getParse(Content content) { + public Map getParse(Content content) { // in memory representation of pdf file PDDocument pdf = null; @@ -84,7 +80,7 @@ && raw.length != Integer.parseInt(contentLength)) { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_TRUNCATED, "Content truncated at "+raw.length - +" bytes. Parser can't handle incomplete pdf file.").getEmptyParse(getConf()); + +" bytes. Parser can't handle incomplete pdf file.").getEmptyParseMap(getConf()); } PDFParser parser = new PDFParser(new ByteArrayInputStream(raw)); @@ -121,17 +117,17 @@ } catch (CryptographyException e) { return new ParseStatus(ParseStatus.FAILED, - "Error decrypting document. " + e).getEmptyParse(getConf()); + "Error decrypting document. " + e).getEmptyParseMap(getConf()); } catch (InvalidPasswordException e) { return new ParseStatus(ParseStatus.FAILED, - "Can't decrypt document - invalid password. " + e).getEmptyParse(getConf()); + "Can't decrypt document - invalid password. " + e).getEmptyParseMap(getConf()); } catch (Exception e) { // run time exception if (LOG.isWarnEnabled()) { LOG.warn("General exception in PDF parser: "+e.getMessage()); e.printStackTrace(LogUtil.getWarnStream(LOG)); } return new ParseStatus(ParseStatus.FAILED, - "Can't be handled as pdf document. " + e).getEmptyParse(getConf()); + "Can't be handled as pdf document. " + e).getEmptyParseMap(getConf()); } finally { try { if (pdf != null) @@ -154,7 +150,7 @@ outlinks, content.getMetadata(), metadata); parseData.setConf(this.conf); - return new ParseImpl(text, parseData); + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); // any filter? //return HtmlParseFilters.filter(content, parse, root); } Index: src/plugin/parse-oo/src/test/org/apache/nutch/parse/oo/TestOOParser.java =================================================================== --- src/plugin/parse-oo/src/test/org/apache/nutch/parse/oo/TestOOParser.java (revision 505236) +++ src/plugin/parse-oo/src/test/org/apache/nutch/parse/oo/TestOOParser.java (working copy) @@ -27,6 +27,7 @@ import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseException; +import org.apache.nutch.parse.ParseImpl; import org.apache.nutch.util.NutchConfiguration; import junit.framework.TestCase; @@ -90,7 +91,7 @@ protocol = factory.getProtocol(urlString); content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); - parse = parser.getParse(content); + parse = ParseImpl.getFirstParseEntry(parser.getParse(content)); String text = parse.getText().replaceAll("[ \t\r\n]+", " "); assertTrue(expectedText.equals(text)); Index: src/plugin/parse-oo/src/java/org/apache/nutch/parse/oo/OOParser.java =================================================================== --- src/plugin/parse-oo/src/java/org/apache/nutch/parse/oo/OOParser.java (revision 505236) +++ src/plugin/parse-oo/src/java/org/apache/nutch/parse/oo/OOParser.java (working copy) @@ -17,25 +17,35 @@ package org.apache.nutch.parse.oo; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.FilterInputStream; import java.net.MalformedURLException; import java.util.ArrayList; import java.util.List; -import java.util.zip.*; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.conf.Configuration; import org.apache.nutch.metadata.Metadata; -import org.apache.nutch.parse.*; +import org.apache.nutch.parse.Outlink; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.parse.ParseImpl; +import org.apache.nutch.parse.ParseStatus; +import org.apache.nutch.parse.Parser; import org.apache.nutch.protocol.Content; import org.apache.nutch.util.LogUtil; import org.apache.nutch.util.NutchConfiguration; -import org.jaxen.*; +import org.jaxen.XPath; import org.jaxen.jdom.JDOMXPath; -import org.jdom.*; -import org.jdom.input.*; +import org.jdom.Document; +import org.jdom.Element; +import org.jdom.Namespace; +import org.jdom.input.SAXBuilder; /** * Parser for OpenOffice and OpenDocument formats. This should handle @@ -60,7 +70,7 @@ return conf; } - public Parse getParse(Content content) { + public Map getParse(Content content) { String text = null; String title = null; Metadata metadata = new Metadata(); @@ -73,7 +83,7 @@ && raw.length != Integer.parseInt(contentLength)) { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_TRUNCATED, "Content truncated at "+raw.length - +" bytes. Parser can't handle incomplete files.").getEmptyParse(conf); + +" bytes. Parser can't handle incomplete files.").getEmptyParseMap(conf); } ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(raw)); ZipEntry ze = null; @@ -88,7 +98,7 @@ } catch (Exception e) { // run time exception e.printStackTrace(LogUtil.getWarnStream(LOG)); return new ParseStatus(ParseStatus.FAILED, - "Can't be handled as OO document. " + e).getEmptyParse(conf); + "Can't be handled as OO document. " + e).getEmptyParseMap(conf); } title = metadata.get(Metadata.TITLE); @@ -100,7 +110,7 @@ Outlink[] links = (Outlink[])outlinks.toArray(new Outlink[outlinks.size()]); ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, links, metadata); - return new ParseImpl(text, parseData); + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); } // extract as much plain text as possible. @@ -206,7 +216,7 @@ fis.read(bytes); fis.close(); Content c = new Content("local", "local", bytes, "application/vnd.oasis.opendocument.text", new Metadata(), conf); - Parse p = oo.getParse(c); + Parse p = ParseImpl.getFirstParseEntry(oo.getParse(c)); System.out.println(p.getData()); System.out.println("Text: '" + p.getText() + "'"); /* Index: src/plugin/parse-text/src/java/org/apache/nutch/parse/text/TextParser.java =================================================================== --- src/plugin/parse-text/src/java/org/apache/nutch/parse/text/TextParser.java (revision 505236) +++ src/plugin/parse-text/src/java/org/apache/nutch/parse/text/TextParser.java (working copy) @@ -17,16 +17,22 @@ package org.apache.nutch.parse.text; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.parse.*; -import org.apache.nutch.util.*; +import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.parse.OutlinkExtractor; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.parse.ParseImpl; +import org.apache.nutch.parse.ParseStatus; +import org.apache.nutch.parse.Parser; +import org.apache.nutch.protocol.Content; +import org.apache.nutch.util.StringUtil; public class TextParser implements Parser { private Configuration conf; - public Parse getParse(Content content) { + public Map getParse(Content content) { // ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, "", new // Outlink[0], metadata); @@ -38,7 +44,7 @@ try { // try to use named encoding text = new String(content.getContent(), encoding); } catch (java.io.UnsupportedEncodingException e) { - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } } else { // FIXME: implement charset detector. This code causes problem when @@ -48,8 +54,8 @@ ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, "", OutlinkExtractor.getOutlinks(text, getConf()), content.getMetadata()); parseData.setConf(this.conf); - return new ParseImpl(text, parseData); - + + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); } public void setConf(Configuration conf) { Index: src/plugin/parse-rtf/src/java/org/apache/nutch/parse/rtf/RTFParseFactory.java =================================================================== --- src/plugin/parse-rtf/src/java/org/apache/nutch/parse/rtf/RTFParseFactory.java (revision 505236) +++ src/plugin/parse-rtf/src/java/org/apache/nutch/parse/rtf/RTFParseFactory.java (working copy) @@ -49,7 +49,7 @@ private Configuration conf; - public Parse getParse(Content content) { + public Map getParse(Content content) { byte[] raw = content.getContent(); Reader reader = new InputStreamReader(new ByteArrayInputStream(raw)); RTFParserDelegateImpl delegate = new RTFParserDelegateImpl(); @@ -63,7 +63,7 @@ } catch (ParseException e) { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_EXCEPTION, - e.toString()).getEmptyParse(conf); + e.toString()).getEmptyParseMap(conf); } Metadata metadata = new Metadata(); @@ -78,13 +78,9 @@ String text = delegate.getText(); - return new ParseImpl(text, - new ParseData(ParseStatus.STATUS_SUCCESS, - title, - OutlinkExtractor - . getOutlinks(text, this.conf), - content.getMetadata(), - metadata)); + ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, OutlinkExtractor + . getOutlinks(text, this.conf), content.getMetadata(), metadata); + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); } public void setConf(Configuration conf) { Index: src/plugin/parse-mp3/src/java/org/apache/nutch/parse/mp3/MP3Parser.java =================================================================== --- src/plugin/parse-mp3/src/java/org/apache/nutch/parse/mp3/MP3Parser.java (revision 505236) +++ src/plugin/parse-mp3/src/java/org/apache/nutch/parse/mp3/MP3Parser.java (working copy) @@ -55,7 +55,7 @@ private MetadataCollector metadataCollector; private Configuration conf; - public Parse getParse(Content content) { + public Parse[] getParse(Content content) { Parse parse = null; byte[] raw = content.getContent(); @@ -75,20 +75,21 @@ } else { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_MISSING_CONTENT, - "No textual content available").getEmptyParse(conf); + "No textual content available").getEmptyParseArray(conf); } } catch (IOException e) { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_EXCEPTION, - "Couldn't create temporary file:" + e).getEmptyParse(conf); + "Couldn't create temporary file:" + e).getEmptyParseArray(conf); } catch (TagException e) { return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_EXCEPTION, - "ID3 Tags could not be parsed:" + e).getEmptyParse(conf); + "ID3 Tags could not be parsed:" + e).getEmptyParseArray(conf); } finally{ tmp.delete(); } - return parse; + Parse[] parses = {parse}; + return parses; } private Parse getID3v1Parse(MP3File mp3, Metadata contentMeta) Index: src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java =================================================================== --- src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java (revision 505236) +++ src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java (working copy) @@ -17,31 +17,27 @@ package org.apache.nutch.parse.ext; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.parse.ParseStatus; -import org.apache.nutch.parse.Parser; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Hashtable; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.net.protocols.Response; +import org.apache.nutch.parse.Outlink; +import org.apache.nutch.parse.OutlinkExtractor; import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseData; import org.apache.nutch.parse.ParseImpl; -import org.apache.nutch.parse.Outlink; -import org.apache.nutch.parse.OutlinkExtractor; - -import org.apache.nutch.util.CommandRunner; -import org.apache.nutch.metadata.Metadata; -import org.apache.nutch.net.protocols.Response; -import org.apache.hadoop.conf.Configuration; - +import org.apache.nutch.parse.ParseStatus; +import org.apache.nutch.parse.Parser; import org.apache.nutch.plugin.Extension; import org.apache.nutch.plugin.PluginRepository; +import org.apache.nutch.protocol.Content; +import org.apache.nutch.util.CommandRunner; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.util.Hashtable; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; - /** * A wrapper that invokes external command to do real parsing job. * @@ -65,14 +61,14 @@ public ExtParser () { } - public Parse getParse(Content content) { + public Map getParse(Content content) { String contentType = content.getContentType(); String[] params = (String[]) TYPE_PARAMS_MAP.get(contentType); if (params == null) return new ParseStatus(ParseStatus.FAILED, - "No external command defined for contentType: " + contentType).getEmptyParse(getConf()); + "No external command defined for contentType: " + contentType).getEmptyParseMap(getConf()); String command = params[0]; int timeout = Integer.parseInt(params[1]); @@ -94,7 +90,7 @@ return new ParseStatus(ParseStatus.FAILED, ParseStatus.FAILED_TRUNCATED, "Content truncated at " + raw.length +" bytes. Parser can't handle incomplete " - + contentType + " file.").getEmptyParse(getConf()); + + contentType + " file.").getEmptyParseMap(getConf()); } ByteArrayOutputStream os = new ByteArrayOutputStream(BUFFER_SIZE); @@ -114,12 +110,12 @@ if (cr.getExitValue() != 0) return new ParseStatus(ParseStatus.FAILED, "External command " + command - + " failed with error: " + es.toString()).getEmptyParse(getConf()); + + " failed with error: " + es.toString()).getEmptyParseMap(getConf()); text = os.toString(); } catch (Exception e) { // run time exception - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } if (text == null) @@ -134,7 +130,7 @@ ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, content.getMetadata()); parseData.setConf(this.conf); - return new ParseImpl(text, parseData); + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); } public void setConf(Configuration conf) { Index: src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java =================================================================== --- src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java (revision 505236) +++ src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java (working copy) @@ -17,27 +17,41 @@ package org.apache.nutch.parse.html; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; -import java.net.URL; -import java.net.MalformedURLException; -import java.io.*; -import java.util.regex.*; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; -import org.cyberneko.html.parsers.*; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.w3c.dom.*; -import org.apache.html.dom.*; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - +import org.apache.hadoop.conf.Configuration; +import org.apache.html.dom.HTMLDocumentImpl; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.net.protocols.Response; +import org.apache.nutch.parse.HTMLMetaTags; +import org.apache.nutch.parse.HtmlParseFilters; +import org.apache.nutch.parse.Outlink; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.parse.ParseImpl; +import org.apache.nutch.parse.ParseStatus; +import org.apache.nutch.parse.Parser; import org.apache.nutch.protocol.Content; -import org.apache.hadoop.conf.*; -import org.apache.nutch.parse.*; -import org.apache.nutch.util.*; +import org.apache.nutch.util.LogUtil; +import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.StringUtil; +import org.cyberneko.html.parsers.DOMFragmentParser; +import org.w3c.dom.DOMException; +import org.w3c.dom.DocumentFragment; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; public class HtmlParser implements Parser { public static final Log LOG = LogFactory.getLog("org.apache.nutch.parse.html"); @@ -101,14 +115,14 @@ private HtmlParseFilters htmlParseFilters; - public Parse getParse(Content content) { + public Map getParse(Content content) { HTMLMetaTags metaTags = new HTMLMetaTags(); URL base; try { base = new URL(content.getBaseUrl()); } catch (MalformedURLException e) { - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } String text = ""; @@ -164,14 +178,14 @@ if (LOG.isTraceEnabled()) { LOG.trace("Parsing..."); } root = parse(input); } catch (IOException e) { - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } catch (DOMException e) { - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } catch (SAXException e) { - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } catch (Exception e) { e.printStackTrace(LogUtil.getWarnStream(LOG)); - return new ParseStatus(e).getEmptyParse(getConf()); + return new ParseStatus(e).getEmptyParseMap(getConf()); } // get meta directives @@ -214,10 +228,10 @@ ParseData parseData = new ParseData(status, title, outlinks, content.getMetadata(), metadata); parseData.setConf(this.conf); - Parse parse = new ParseImpl(text, parseData); + Map parseMap = ParseImpl.createSingleEntryMap(text, parseData, base.toString()); // run filters on parse - return this.htmlParseFilters.filter(content, parse, metaTags, root); + return this.htmlParseFilters.filter(content, parseMap, metaTags, root); } private DocumentFragment parse(InputSource input) throws Exception { @@ -287,8 +301,8 @@ Configuration conf = NutchConfiguration.create(); HtmlParser parser = new HtmlParser(); parser.setConf(conf); - Parse parse = parser.getParse( - new Content(url, url, bytes, "text/html", new Metadata(), conf)); + Parse parse = ParseImpl.getFirstParseEntry(parser.getParse( + new Content(url, url, bytes, "text/html", new Metadata(), conf))); System.out.println("data: "+parse.getData()); System.out.println("text: "+parse.getText()); Index: src/plugin/parse-mspowerpoint/src/java/org/apache/nutch/parse/mspowerpoint/MSPowerPointParser.java =================================================================== --- src/plugin/parse-mspowerpoint/src/java/org/apache/nutch/parse/mspowerpoint/MSPowerPointParser.java (revision 505236) +++ src/plugin/parse-mspowerpoint/src/java/org/apache/nutch/parse/mspowerpoint/MSPowerPointParser.java (working copy) @@ -16,6 +16,8 @@ */ package org.apache.nutch.parse.mspowerpoint; +import java.util.Map; + // Nutch imports import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ms.MSBaseParser; @@ -41,7 +43,7 @@ public static final String MIME_TYPE = "application/vnd.ms-powerpoint"; - public Parse getParse(final Content content) { + public Map getParse(final Content content) { return getParse(new PPTExtractor(), content); } Index: src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java =================================================================== --- src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java (revision 505236) +++ src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java (working copy) @@ -20,16 +20,12 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; +import java.util.Map; import java.util.Properties; -// Commons Logging imports import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - -// Hadoop imports import org.apache.hadoop.conf.Configuration; - -// Nutch imports import org.apache.nutch.metadata.DublinCore; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.net.protocols.Response; @@ -61,7 +57,7 @@ * Parses a Content with a specific {@link MSExtractor Microsoft document * extractor}. */ - protected Parse getParse(MSExtractor extractor, Content content) { + protected Map getParse(MSExtractor extractor, Content content) { String text = null; String title = null; @@ -77,7 +73,7 @@ ParseStatus.FAILED_TRUNCATED, "Content truncated at " + raw.length +" bytes. " + "Parser can't handle incomplete file.") - .getEmptyParse(this.conf); + .getEmptyParseMap(this.conf); } extractor.extract(new ByteArrayInputStream(raw)); text = extractor.getText(); @@ -87,7 +83,7 @@ } catch (Exception e) { return new ParseStatus(ParseStatus.FAILED, "Can't be handled as Microsoft document. " + e) - .getEmptyParse(this.conf); + .getEmptyParseMap(this.conf); } // collect meta data @@ -105,7 +101,7 @@ outlinks, content.getMetadata(), metadata); parseData.setConf(this.conf); - return new ParseImpl(text, parseData); + return ParseImpl.createSingleEntryMap(text, parseData, content.getUrl()); //FIXME or use content.getBaseUrl()?? } @@ -127,7 +123,7 @@ Content content = new Content(file, file, raw, mime, meta, NutchConfiguration.create()); - System.out.println(parser.getParse(content).getText()); + System.out.println(ParseImpl.getFirstParseEntry(parser.getParse(content)).getText()); } private final static byte[] getRawBytes(File f) {