Index: src/java/org/apache/nutch/indexer/solr/SolrWriter.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrWriter.java (revision 982590) +++ src/java/org/apache/nutch/indexer/solr/SolrWriter.java (working copy) @@ -23,6 +23,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.indexer.NutchField; import org.apache.nutch.indexer.NutchIndexWriter; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; @@ -47,16 +48,16 @@ public void write(NutchDocument doc) throws IOException { final SolrInputDocument inputDoc = new SolrInputDocument(); - for(final Entry> e : doc) { - for (final String val : e.getValue()) { - inputDoc.addField(solrMapping.mapKey(e.getKey()), val); + for(final Entry e : doc) { + for (final Object val : e.getValue().getValues()) { + inputDoc.addField(solrMapping.mapKey(e.getKey()), val, e.getValue().getWeight()); String sCopy = solrMapping.mapCopyKey(e.getKey()); if (sCopy != e.getKey()) { - inputDoc.addField(sCopy, val); + inputDoc.addField(sCopy, val, e.getValue().getWeight()); } } } - inputDoc.setDocumentBoost(doc.getScore()); + inputDoc.setDocumentBoost(doc.getWeight()); inputDocs.add(inputDoc); if (inputDocs.size() > commitSize) { try { Index: src/java/org/apache/nutch/indexer/NutchDocument.java =================================================================== --- src/java/org/apache/nutch/indexer/NutchDocument.java (revision 982590) +++ src/java/org/apache/nutch/indexer/NutchDocument.java (working copy) @@ -19,11 +19,9 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -35,51 +33,48 @@ /** A {@link NutchDocument} is the unit of indexing.*/ public class NutchDocument -implements Writable, Iterable>> { +implements Writable, Iterable> { - public static final byte VERSION = 1; + public static final byte VERSION = 2; + + private Map fields; - private Map> fields; - private Metadata documentMeta; - private float score; + private float weight; public NutchDocument() { - fields = new HashMap>(); + fields = new HashMap(); documentMeta = new Metadata(); - score = 0.0f; + weight = 1.0f; } - public void add(String name, String value) { - List fieldValues = fields.get(name); - if (fieldValues == null) { - fieldValues = new ArrayList(); + public void add(String name, Object value) { + NutchField field = fields.get(name); + if (field == null) { + field = new NutchField(value); + fields.put(name, field); + } else { + field.add(value); } - fieldValues.add(value); - fields.put(name, fieldValues); } - private void addFieldUnprotected(String name, String value) { - fields.get(name).add(value); - } - - public String getFieldValue(String name) { - List fieldValues = fields.get(name); - if (fieldValues == null) { + public Object getFieldValue(String name) { + NutchField field = fields.get(name); + if (field == null) { return null; } - if (fieldValues.size() == 0) { + if (field.getValues().size() == 0) { return null; } - return fieldValues.get(0); + return field.getValues().get(0); } - public List getFieldValues(String name) { + public NutchField getField(String name) { return fields.get(name); } - public List removeField(String name) { + public NutchField removeField(String name) { return fields.remove(name); } @@ -88,16 +83,16 @@ } /** Iterate over all fields. */ - public Iterator>> iterator() { + public Iterator> iterator() { return fields.entrySet().iterator(); } - public float getScore() { - return score; + public float getWeight() { + return weight; } - public void setScore(float score) { - this.score = score; + public void setWeight(float weight) { + this.weight = weight; } public Metadata getDocumentMeta() { @@ -105,6 +100,7 @@ } public void readFields(DataInput in) throws IOException { + fields.clear(); byte version = in.readByte(); if (version != VERSION) { throw new VersionMismatchException(VERSION, version); @@ -112,30 +108,23 @@ int size = WritableUtils.readVInt(in); for (int i = 0; i < size; i++) { String name = Text.readString(in); - int numValues = WritableUtils.readVInt(in); - fields.put(name, new ArrayList()); - for (int j = 0; j < numValues; j++) { - String value = Text.readString(in); - addFieldUnprotected(name, value); - } + NutchField field = new NutchField(); + field.readFields(in); + fields.put(name, field); } - score = in.readFloat(); + weight = in.readFloat(); documentMeta.readFields(in); } public void write(DataOutput out) throws IOException { out.writeByte(VERSION); WritableUtils.writeVInt(out, fields.size()); - for (Map.Entry> entry : fields.entrySet()) { + for (Map.Entry entry : fields.entrySet()) { Text.writeString(out, entry.getKey()); - List values = entry.getValue(); - WritableUtils.writeVInt(out, values.size()); - for (String value : values) { - Text.writeString(out, value); - } + NutchField field = entry.getValue(); + field.write(out); } - out.writeFloat(score); + out.writeFloat(weight); documentMeta.write(out); } - } Index: src/java/org/apache/nutch/indexer/IndexerMapReduce.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexerMapReduce.java (revision 982590) +++ src/java/org/apache/nutch/indexer/IndexerMapReduce.java (working copy) @@ -152,7 +152,7 @@ return; } // apply boost to all indexed fields. - doc.setScore(boost); + doc.setWeight(boost); // store boost for use by explain and dedup doc.add("boost", Float.toString(boost)); Index: src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java =================================================================== --- src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java (revision 982590) +++ src/java/org/apache/nutch/indexer/lucene/LuceneWriter.java (working copy) @@ -40,6 +40,7 @@ import org.apache.nutch.analysis.NutchDocumentAnalyzer; import org.apache.nutch.indexer.Indexer; import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.indexer.NutchField; import org.apache.nutch.indexer.NutchIndexWriter; import org.apache.nutch.indexer.NutchSimilarity; import org.apache.nutch.metadata.Metadata; @@ -78,10 +79,10 @@ private Document createLuceneDoc(NutchDocument doc) { final Document out = new Document(); - out.setBoost(doc.getScore()); + out.setBoost(doc.getWeight()); final Metadata documentMeta = doc.getDocumentMeta(); - for (final Entry> entry : doc) { + for (final Entry entry : doc) { final String fieldName = entry.getKey(); Field.Store store = fieldStore.get(fieldName); @@ -132,8 +133,10 @@ } } - for (final String fieldValue : entry.getValue()) { - out.add(new Field(fieldName, fieldValue, store, index, vector)); + for (final Object fieldValue : entry.getValue().getValues()) { + Field f = new Field(fieldName, fieldValue.toString(), store, index, vector); + f.setBoost(entry.getValue().getWeight()); + out.add(f); } } Index: src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java (revision 982590) +++ src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java (working copy) @@ -26,6 +26,7 @@ import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.Inlinks; import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.indexer.NutchField; import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseData; import org.apache.nutch.protocol.Content; @@ -52,12 +53,12 @@ CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) throws ScoringFilterException { - List tlds = doc.getFieldValues("tld"); + NutchField tlds = doc.getField("tld"); float boost = 1.0f; if(tlds != null) { - for(String tld : tlds) { - DomainSuffix entry = tldEntries.get(tld); + for(Object tld : tlds.getValues()) { + DomainSuffix entry = tldEntries.get(tld.toString()); if(entry != null) boost *= entry.getBoost(); }