Index: src/test/org/apache/nutch/searcher/QueryContextTest.java =================================================================== --- src/test/org/apache/nutch/searcher/QueryContextTest.java (revision 0) +++ src/test/org/apache/nutch/searcher/QueryContextTest.java (revision 0) @@ -0,0 +1,15 @@ +package org.apache.nutch.searcher; + +import junit.framework.TestCase; + +import org.apache.nutch.util.WritableTestUtils; + +public class QueryContextTest extends TestCase { + + public void testWritable() throws Exception { + QueryContext context = new QueryContext(10, 2, "site", null, false); + context.put("cat", "dog"); + WritableTestUtils.testWritable(context); + } + +} Index: src/java/org/apache/nutch/searcher/Query.java =================================================================== --- src/java/org/apache/nutch/searcher/Query.java (revision 894348) +++ src/java/org/apache/nutch/searcher/Query.java (working copy) @@ -41,6 +41,16 @@ public final class Query implements Writable, Cloneable, Configurable { public static final Log LOG = LogFactory.getLog(Query.class); + private QueryContext context; + + public void setContext(QueryContext context) { + this.context = context; + } + + public QueryContext getContext() { + return context; + } + /** A query clause. */ public static class Clause implements Cloneable { public static final String DEFAULT_FIELD = "DEFAULT"; @@ -293,6 +303,7 @@ public Query(Configuration conf) { this.conf = conf; + this.context = new QueryContext(); } public void setConf(Configuration conf) { @@ -362,6 +373,7 @@ out.writeByte(clauses.size()); for (int i = 0; i < clauses.size(); i++) clauses.get(i).write(out); + context.write(out); } public static Query read(DataInput in, Configuration conf) throws IOException { @@ -375,6 +387,8 @@ int length = in.readByte(); for (int i = 0; i < length; i++) clauses.add(Clause.read(in, this.conf)); + + context.readFields(in); } public String toString() { @@ -390,7 +404,7 @@ public boolean equals(Object o) { if (!(o instanceof Query)) return false; Query other = (Query)o; - return this.clauses.equals(other.clauses); + return this.clauses.equals(other.clauses) && this.context.equals(other.context); } public int hashCode() { Index: src/java/org/apache/nutch/searcher/LuceneSearchBean.java =================================================================== --- src/java/org/apache/nutch/searcher/LuceneSearchBean.java (revision 894348) +++ src/java/org/apache/nutch/searcher/LuceneSearchBean.java (working copy) @@ -78,11 +78,20 @@ } } + + @Override + @Deprecated public Hits search(Query query, int numHits, String dedupField, String sortField, boolean reverse) throws IOException { - return searcher.search(query, numHits, dedupField, sortField, reverse); + query.setContext(new QueryContext(numHits, QueryContext.DEFAULT_MAX_HITS_PER_DUP, dedupField, sortField, reverse)); + return searcher.search(query); } + + @Override + public Hits search(Query query) throws IOException { + return searcher.search(query); + } public String getExplanation(Query query, Hit hit) throws IOException { return searcher.getExplanation(query, hit); Index: src/java/org/apache/nutch/searcher/SolrSearchBean.java =================================================================== --- src/java/org/apache/nutch/searcher/SolrSearchBean.java (revision 894348) +++ src/java/org/apache/nutch/searcher/SolrSearchBean.java (working copy) @@ -68,24 +68,23 @@ return "SOLR backend does not support explanations yet."; } - @SuppressWarnings("unchecked") - public Hits search(Query query, int numHits, String dedupField, - String sortField, boolean reverse) - throws IOException { - + + public Hits search(Query query) throws IOException { // filter query string final BooleanQuery bQuery = filters.filter(query); final SolrQuery solrQuery = new SolrQuery(stringify(bQuery)); - solrQuery.setRows(numHits); + solrQuery.setRows(query.getContext().getNumHits()); - if (sortField == null) { - solrQuery.setFields(dedupField, "score", searchUID); - sortField = "score"; + if (query.getContext().getSortField() == null) { + solrQuery.setFields(query.getContext().getDedupField(), "score", searchUID); + query.getContext().setSortField("score"); } else { - solrQuery.setFields(dedupField, sortField, searchUID); - solrQuery.setSortField(sortField, reverse ? ORDER.asc : ORDER.desc); + solrQuery.setFields(query.getContext().getDedupField(), query + .getContext().getSortField(), searchUID); + solrQuery.setSortField(query.getContext().getSortField(), query + .getContext().isReverse() ? ORDER.asc : ORDER.desc); } QueryResponse response; @@ -101,7 +100,7 @@ for (int i = 0; i < hitArr.length; i++) { final SolrDocument solrDoc = docList.get(i); - final Object raw = solrDoc.getFirstValue(sortField); + final Object raw = solrDoc.getFirstValue(query.getContext().getSortField()); WritableComparable sortValue; if (raw instanceof Integer) { @@ -116,7 +115,7 @@ throw new RuntimeException("Unknown sort value type!"); } - final String dedupValue = (String) solrDoc.getFirstValue(dedupField); + final String dedupValue = (String) solrDoc.getFirstValue(query.getContext().getDedupField()); final String uniqueKey = (String )solrDoc.getFirstValue(searchUID); @@ -126,6 +125,18 @@ return new Hits(docList.getNumFound(), hitArr); } + @SuppressWarnings("unchecked") + @Deprecated + public Hits search(Query query, int numHits, String dedupField, + String sortField, boolean reverse) + throws IOException { + query.getContext().setNumHits(numHits); + query.getContext().setDedupField(dedupField); + query.getContext().setSortField(sortField); + query.getContext().setReverse(reverse); + return search(query); + } + public HitDetails getDetails(Hit hit) throws IOException { QueryResponse response; try { Index: src/java/org/apache/nutch/searcher/Searcher.java =================================================================== --- src/java/org/apache/nutch/searcher/Searcher.java (revision 894348) +++ src/java/org/apache/nutch/searcher/Searcher.java (working copy) @@ -23,11 +23,21 @@ /** Service that searches. */ public interface Searcher extends Closeable { - /** Return the top-scoring hits for a query. */ - Hits search(Query query, int numHits, - String dedupField, String sortField, boolean reverse) - throws IOException; + /** + * Return the top-scoring hits for a query. + * + * @deprecated since 1.1, use {@link #search(Query)} instead. + * */ + Hits search(Query query, int numHits, String dedupField, String sortField, + boolean reverse) throws IOException; + /** + * Return the top-scoring hits for a query. + * + * @since 1.1 + */ + Hits search(Query query) throws IOException; + /** Return an HTML-formatted explanation of how a query scored. */ String getExplanation(Query query, Hit hit) throws IOException; } Index: src/java/org/apache/nutch/searcher/NutchBean.java =================================================================== --- src/java/org/apache/nutch/searcher/NutchBean.java (revision 894348) +++ src/java/org/apache/nutch/searcher/NutchBean.java (working copy) @@ -151,86 +151,34 @@ return segmentBean.getSegmentNames(); } + /** + * @deprecated since 1.1, use {@link #search(Query)} instead + */ public Hits search(Query query, int numHits) throws IOException { return search(query, numHits, null, null, false); } + /** + * @deprecated since 1.1, use {@link #search(Query)} instead + */ public Hits search(Query query, int numHits, String dedupField, String sortField, boolean reverse) throws IOException { return searchBean.search(query, numHits, dedupField, sortField, reverse); } + + @Override + public Hits search(Query query) throws IOException { + if (query.getContext().getMaxHitsPerDup() <= 0) // disable dup checking + return searchBean.search(query); - @SuppressWarnings("serial") - private class DupHits extends ArrayList { - private boolean maxSizeExceeded; - } - - /** Search for pages matching a query, eliminating excessive hits from the - * same site. Hits after the first maxHitsPerDup from the same - * site are removed from results. The remaining hits have {@link - * Hit#moreFromDupExcluded()} set.

If maxHitsPerDup is zero then all - * hits are returned. - * - * @param query query - * @param numHits number of requested hits - * @param maxHitsPerDup the maximum hits returned with matching values, or zero - * @return Hits the matching hits - * @throws IOException - */ - public Hits search(Query query, int numHits, int maxHitsPerDup) - throws IOException { - return search(query, numHits, maxHitsPerDup, "site", null, false); - } - - /** Search for pages matching a query, eliminating excessive hits with - * matching values for a named field. Hits after the first - * maxHitsPerDup are removed from results. The remaining hits - * have {@link Hit#moreFromDupExcluded()} set.

If maxHitsPerDup is zero - * then all hits are returned. - * - * @param query query - * @param numHits number of requested hits - * @param maxHitsPerDup the maximum hits returned with matching values, or zero - * @param dedupField field name to check for duplicates - * @return Hits the matching hits - * @throws IOException - */ - public Hits search(Query query, int numHits, - int maxHitsPerDup, String dedupField) - throws IOException { - return search(query, numHits, maxHitsPerDup, dedupField, null, false); - } - /** Search for pages matching a query, eliminating excessive hits with - * matching values for a named field. Hits after the first - * maxHitsPerDup are removed from results. The remaining hits - * have {@link Hit#moreFromDupExcluded()} set.

If maxHitsPerDup is zero - * then all hits are returned. - * - * @param query query - * @param numHits number of requested hits - * @param maxHitsPerDup the maximum hits returned with matching values, or zero - * @param dedupField field name to check for duplicates - * @param sortField Field to sort on (or null if no sorting). - * @param reverse True if we are to reverse sort by sortField. - * @return Hits the matching hits - * @throws IOException - */ - public Hits search(Query query, int numHits, - int maxHitsPerDup, String dedupField, - String sortField, boolean reverse) - throws IOException { - if (maxHitsPerDup <= 0) // disable dup checking - return search(query, numHits, dedupField, sortField, reverse); - final float rawHitsFactor = this.conf.getFloat("searcher.hostgrouping.rawhits.factor", 2.0f); - int numHitsRaw = (int)(numHits * rawHitsFactor); + int numHitsRaw = (int)(query.getContext().getNumHits() * rawHitsFactor); if (LOG.isInfoEnabled()) { LOG.info("searching for "+numHitsRaw+" raw hits"); } - Hits hits = searchBean.search(query, numHitsRaw, - dedupField, sortField, reverse); + Hits hits = searchBean.search(query); final long total = hits.getTotal(); final Map dupToHits = new HashMap(); final List resultList = new ArrayList(); @@ -246,14 +194,13 @@ if (i == MAX_PROHIBITED_TERMS) break; optQuery.addProhibitedTerm(excludedValues.get(i), - dedupField); + query.getContext().getDedupField()); } numHitsRaw = (int)(numHitsRaw * rawHitsFactor); if (LOG.isInfoEnabled()) { LOG.info("re-searching for "+numHitsRaw+" raw hits, query: "+optQuery); } - hits = searchBean.search(optQuery, numHitsRaw, - dedupField, sortField, reverse); + hits = searchBean.search(optQuery); if (LOG.isInfoEnabled()) { LOG.info("found "+hits.getTotal()+" raw hits"); } @@ -273,7 +220,7 @@ dupToHits.put(value, dupHits = new DupHits()); // does this hit exceed maxHitsPerDup? - if (dupHits.size() == maxHitsPerDup) { // yes -- ignore the hit + if (dupHits.size() == query.getContext().getMaxHitsPerDup()) { // yes -- ignore the hit if (!dupHits.maxSizeExceeded) { // mark prior hits with moreFromDupExcluded @@ -292,7 +239,7 @@ // are we done? // we need to find one more than asked for, so that we can tell if // there are more hits to be shown - if (resultList.size() > numHits) + if (resultList.size() > query.getContext().getNumHits()) break; } } @@ -304,7 +251,74 @@ return results; } + @SuppressWarnings("serial") + private class DupHits extends ArrayList { + private boolean maxSizeExceeded; + } + /** Search for pages matching a query, eliminating excessive hits from the + * same site. Hits after the first maxHitsPerDup from the same + * site are removed from results. The remaining hits have {@link + * Hit#moreFromDupExcluded()} set.

If maxHitsPerDup is zero then all + * hits are returned. + * + * @param query query + * @param numHits number of requested hits + * @param maxHitsPerDup the maximum hits returned with matching values, or zero + * @return Hits the matching hits + * @throws IOException + * @deprecated since 1.1, use {@link #search(Query)} instead + * + */ + public Hits search(Query query, int numHits, int maxHitsPerDup) + throws IOException { + return search(query, numHits, maxHitsPerDup, "site", null, false); + } + + /** Search for pages matching a query, eliminating excessive hits with + * matching values for a named field. Hits after the first + * maxHitsPerDup are removed from results. The remaining hits + * have {@link Hit#moreFromDupExcluded()} set.

If maxHitsPerDup is zero + * then all hits are returned. + * + * @param query query + * @param numHits number of requested hits + * @param maxHitsPerDup the maximum hits returned with matching values, or zero + * @param dedupField field name to check for duplicates + * @return Hits the matching hits + * @throws IOException + * @deprecated since 1.1, use {@link #search(Query)} instead + */ + public Hits search(Query query, int numHits, + int maxHitsPerDup, String dedupField) + throws IOException { + return search(query, numHits, maxHitsPerDup, dedupField, null, false); + } + /** Search for pages matching a query, eliminating excessive hits with + * matching values for a named field. Hits after the first + * maxHitsPerDup are removed from results. The remaining hits + * have {@link Hit#moreFromDupExcluded()} set.

If maxHitsPerDup is zero + * then all hits are returned. + * + * @param query query + * @param numHits number of requested hits + * @param maxHitsPerDup the maximum hits returned with matching values, or zero + * @param dedupField field name to check for duplicates + * @param sortField Field to sort on (or null if no sorting). + * @param reverse True if we are to reverse sort by sortField. + * @return Hits the matching hits + * @throws IOException + * @deprecated since 1.1, use {@link #search(Query)} instead + */ + public Hits search(Query query, int numHits, + int maxHitsPerDup, String dedupField, + String sortField, boolean reverse) + throws IOException { + query.setContext(new QueryContext(numHits, maxHitsPerDup, dedupField, sortField, reverse)); + return search(query); + } + + public String getExplanation(Query query, Hit hit) throws IOException { return searchBean.getExplanation(query, hit); } Index: src/java/org/apache/nutch/searcher/QueryContext.java =================================================================== --- src/java/org/apache/nutch/searcher/QueryContext.java (revision 0) +++ src/java/org/apache/nutch/searcher/QueryContext.java (revision 0) @@ -0,0 +1,133 @@ +package org.apache.nutch.searcher; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.nutch.metadata.Metadata; + +/** + * Query context object that describes the context of the query. + */ +public class QueryContext implements Writable { + + public static final String DEFAULT_DEDUP_FIELD = "site"; + public static final int DEFAULT_MAX_HITS_PER_DUP = 2; + public static final int DEFAULT_NUM_HITS = 10; + public static final boolean DEFAULT_REVERSE = false; + + private Metadata metadata = new Metadata(); + + public void setNumHits(int numHits) { + this.numHits = numHits; + } + + public void setMaxHitsPerDup(int maxHitsPerDup) { + this.maxHitsPerDup = maxHitsPerDup; + } + + public void setDedupField(String dedupField) { + this.dedupField = dedupField; + } + + public void setSortField(String sortField) { + this.sortField = sortField; + } + + public void setReverse(boolean reverse) { + this.reverse = reverse; + } + + private int numHits; + private int maxHitsPerDup; + private String dedupField; + private String sortField; + private boolean reverse; + + public QueryContext() { + setNumHits(DEFAULT_NUM_HITS); + setMaxHitsPerDup(DEFAULT_MAX_HITS_PER_DUP); + setDedupField(DEFAULT_DEDUP_FIELD); + setSortField(sortField); + setReverse(false); + + } + + public QueryContext(int numHits, int maxHitsPerDup, String dedupField, + String sortField, boolean reverse) { + setNumHits(numHits); + setMaxHitsPerDup(maxHitsPerDup); + setDedupField(dedupField); + setSortField(sortField); + setReverse(reverse); + } + + public int getNumHits() { + return numHits; + } + + public int getMaxHitsPerDup() { + return maxHitsPerDup; + } + + public String getDedupField() { + return dedupField; + } + + public String getSortField() { + return sortField; + } + + public boolean isReverse() { + return reverse; + } + + public String get(String name) { + return metadata.get(name); + } + + public void put(String name, String value) { + metadata.set(name, value); + } + + @Override + public void readFields(DataInput input) throws IOException { + metadata.readFields(input); + numHits = WritableUtils.readVInt(input); + maxHitsPerDup = WritableUtils.readVInt(input); + dedupField = WritableUtils.readString(input); + sortField = WritableUtils.readString(input); + reverse = input.readBoolean(); + } + + @Override + public void write(DataOutput output) throws IOException { + metadata.write(output); + WritableUtils.writeVInt(output, numHits); + WritableUtils.writeVInt(output, maxHitsPerDup); + WritableUtils.writeString(output, dedupField); + WritableUtils.writeString(output, sortField); + output.writeBoolean(reverse); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof QueryContext) { + + QueryContext other = (QueryContext) obj; + return other.numHits == this.numHits + && other.metadata.equals(this.metadata) + && other.reverse == this.reverse + && other.maxHitsPerDup == this.maxHitsPerDup + && ((other.dedupField != null && other.dedupField + .equals(this.dedupField)) || (other.dedupField == null && this.dedupField == null)) + && ((other.sortField != null && other.sortField + .equals(this.sortField)) || (other.sortField == null && this.sortField == null)); + + } else { + return false; + } + } +} Index: src/java/org/apache/nutch/searcher/IndexSearcher.java =================================================================== --- src/java/org/apache/nutch/searcher/IndexSearcher.java (revision 894348) +++ src/java/org/apache/nutch/searcher/IndexSearcher.java (working copy) @@ -89,18 +89,28 @@ } } + @Override + @Deprecated public Hits search(Query query, int numHits, String dedupField, String sortField, boolean reverse) throws IOException { + query.setContext(new QueryContext(numHits, + QueryContext.DEFAULT_MAX_HITS_PER_DUP, dedupField, sortField, reverse)); + return search(query); + } + + @Override + public Hits search(Query query) throws IOException { org.apache.lucene.search.BooleanQuery luceneQuery = this.queryFilters.filter(query); - return translateHits - (optimizer.optimize(luceneQuery, luceneSearcher, numHits, - sortField, reverse), - dedupField, sortField); + return translateHits(optimizer.optimize(luceneQuery, luceneSearcher, query + .getContext().getNumHits(), query.getContext().getSortField(), query + .getContext().isReverse()), query.getContext().getDedupField(), query + .getContext().getSortField()); } + public String getExplanation(Query query, Hit hit) throws IOException { return luceneSearcher.explain(this.queryFilters.filter(query), Integer.valueOf(hit.getUniqueKey())).toHtml(); Index: src/java/org/apache/nutch/searcher/DistributedSearchBean.java =================================================================== --- src/java/org/apache/nutch/searcher/DistributedSearchBean.java (revision 894348) +++ src/java/org/apache/nutch/searcher/DistributedSearchBean.java (working copy) @@ -49,10 +49,6 @@ private int id; private Query query; - private int numHits; - private String dedupField; - private String sortField; - private boolean reverse; public SearchTask(int id) { this.id = id; @@ -62,18 +58,22 @@ if (!liveServers[id]) { return null; } - return beans[id].search(query, numHits, dedupField, sortField, reverse); + return beans[id].search(query); } + /** + * @deprecated since 1.1, use {@link #setSearchArgs(Query)} instead + */ public void setSearchArgs(Query query, int numHits, String dedupField, String sortField, boolean reverse) { this.query = query; - this.numHits = numHits; - this.dedupField = dedupField; - this.sortField = sortField; - this.reverse = reverse; + query.setContext(new QueryContext(numHits, QueryContext.DEFAULT_MAX_HITS_PER_DUP, dedupField, sortField, reverse)); } + public void setSearchArgs(Query query) { + this.query = query; + } + } private class DetailTask implements Callable { @@ -199,12 +199,10 @@ return beans[hit.getIndexNo()].getExplanation(query, hit); } - public Hits search(Query query, int numHits, String dedupField, - String sortField, boolean reverse) throws IOException { - + @Override + public Hits search(Query query) throws IOException { for (Callable task : searchTasks) { - ((SearchTask)task).setSearchArgs(query, numHits, dedupField, sortField, - reverse); + ((SearchTask)task).setSearchArgs(query); } List> allHits; @@ -216,10 +214,12 @@ } PriorityQueue queue; // cull top hits from results - if (sortField == null || reverse) { - queue = new PriorityQueue(numHits); + if (query.getContext().getSortField() == null + || query.getContext().isReverse()) { + queue = new PriorityQueue(query.getContext().getNumHits()); } else { - queue = new PriorityQueue(numHits, new Comparator() { + queue = new PriorityQueue(query.getContext().getNumHits(), + new Comparator() { public int compare(Hit h1, Hit h2) { return h2.compareTo(h1); // reverse natural order } @@ -251,7 +251,8 @@ Hit newHit = new Hit(i, hit.getUniqueKey(), hit.getSortValue(), hit.getDedupValue()); queue.add(newHit); - if (queue.size() > numHits) { // if hit queue overfull + if (queue.size() > query.getContext().getNumHits()) { + // if hit queue overfull queue.remove(); } } @@ -265,6 +266,15 @@ return new Hits(totalHits, culledResults); } + @Override + @Deprecated + public Hits search(Query query, int numHits, String dedupField, + String sortField, boolean reverse) throws IOException { + + query.setContext(new QueryContext(numHits, QueryContext.DEFAULT_MAX_HITS_PER_DUP, dedupField, sortField, reverse)); + return search(query); + } + public void close() throws IOException { executor.shutdown(); pingService.shutdown();