Index: src/java/org/apache/nutch/indexer/solr/SolrConstants.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrConstants.java (revision 1170103) +++ src/java/org/apache/nutch/indexer/solr/SolrConstants.java (working copy) @@ -21,6 +21,8 @@ public static final String SERVER_URL = SOLR_PREFIX + "server.url"; + public static final String DELETE_DB_GONE = SOLR_PREFIX + "delete.gone"; + public static final String COMMIT_SIZE = SOLR_PREFIX + "commit.size"; public static final String MAPPING_FILE = SOLR_PREFIX + "mapping.file"; Index: src/java/org/apache/nutch/indexer/solr/SolrWriter.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrWriter.java (revision 1170103) +++ src/java/org/apache/nutch/indexer/solr/SolrWriter.java (working copy) @@ -44,13 +44,27 @@ new ArrayList(); private int commitSize; + private int numDeletes = 0; + private boolean delete = false; public void open(JobConf job, String name) throws IOException { solr = SolrUtils.getCommonsHttpSolrServer(job); commitSize = job.getInt(SolrConstants.COMMIT_SIZE, 1000); solrMapping = SolrMappingReader.getInstance(job); + delete = job.getBoolean(SolrConstants.DELETE_DB_GONE, false); } + public void delete(String key) throws IOException { + if (delete) { + try { + solr.deleteById(key); + numDeletes++; + } catch (final SolrServerException e) { + throw makeIOException(e); + } + } + } + public void write(NutchDocument doc) throws IOException { final SolrInputDocument inputDoc = new SolrInputDocument(); for(final Entry e : doc) { @@ -75,10 +89,12 @@ } inputDoc.setDocumentBoost(doc.getWeight()); inputDocs.add(inputDoc); - if (inputDocs.size() >= commitSize) { + if (inputDocs.size() + numDeletes >= commitSize) { try { - LOG.info("Adding " + Integer.toString(inputDocs.size()) + " documents"); + LOG.info("Indexing " + Integer.toString(inputDocs.size()) + " documents"); + LOG.info("Deleting " + Integer.toString(numDeletes) + " documents"); solr.add(inputDocs); + numDeletes = 0; } catch (final SolrServerException e) { throw makeIOException(e); } @@ -89,11 +105,13 @@ public void close() throws IOException { try { if (!inputDocs.isEmpty()) { - LOG.info("Adding " + Integer.toString(inputDocs.size()) + " documents"); + LOG.info("Indexing " + Integer.toString(inputDocs.size()) + " documents"); solr.add(inputDocs); inputDocs.clear(); } - // solr.commit(); + if (numDeletes > 0) { + LOG.info("Deleting " + Integer.toString(numDeletes) + " documents"); + } } catch (final SolrServerException e) { throw makeIOException(e); } Index: src/java/org/apache/nutch/indexer/solr/SolrIndexer.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrIndexer.java (revision 1170103) +++ src/java/org/apache/nutch/indexer/solr/SolrIndexer.java (working copy) @@ -57,11 +57,11 @@ public void indexSolr(String solrUrl, Path crawlDb, Path linkDb, List segments) throws IOException { - indexSolr(solrUrl, crawlDb, linkDb, segments, false); + indexSolr(solrUrl, crawlDb, linkDb, segments, false, false); } public void indexSolr(String solrUrl, Path crawlDb, Path linkDb, - List segments, boolean noCommit) throws IOException { + List segments, boolean noCommit, boolean delete) throws IOException { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); long start = System.currentTimeMillis(); LOG.info("SolrIndexer: starting at " + sdf.format(start)); @@ -69,9 +69,15 @@ final JobConf job = new NutchJob(getConf()); job.setJobName("index-solr " + solrUrl); + if (delete) { + LOG.info("SolrIndexer: deleting 404 documents"); + } + IndexerMapReduce.initMRJob(crawlDb, linkDb, segments, job); job.set(SolrConstants.SERVER_URL, solrUrl); + job.setBoolean(SolrConstants.DELETE_DB_GONE, delete); + NutchIndexWriterFactory.addClassToConf(job, SolrWriter.class); job.setReduceSpeculativeExecution(false); @@ -100,7 +106,7 @@ public int run(String[] args) throws Exception { if (args.length < 3) { - System.err.println("Usage: SolrIndexer [-linkdb ] ( ... | -dir ) [-noCommit]"); + System.err.println("Usage: SolrIndexer [-linkdb ] ( ... | -dir ) [-noCommit] [-delete]"); return -1; } @@ -110,6 +116,7 @@ final List segments = new ArrayList(); boolean noCommit = false; + boolean delete = false; for (int i = 2; i < args.length; i++) { if (args[i].equals("-linkdb")) { @@ -126,13 +133,15 @@ } } else if (args[i].equals("-noCommit")) { noCommit = true; + } else if (args[i].equals("-delete")) { + delete = true; } else { segments.add(new Path(args[i])); } } try { - indexSolr(args[0], crawlDb, linkDb, segments, noCommit); + indexSolr(args[0], crawlDb, linkDb, segments, noCommit, delete); return 0; } catch (final Exception e) { LOG.fatal("SolrIndexer: " + StringUtils.stringifyException(e)); Index: src/java/org/apache/nutch/indexer/IndexerMapReduce.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexerMapReduce.java (revision 1170103) +++ src/java/org/apache/nutch/indexer/IndexerMapReduce.java (working copy) @@ -71,11 +71,13 @@ public void reduce(Text key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { + boolean delete = false; Inlinks inlinks = null; CrawlDatum dbDatum = null; CrawlDatum fetchDatum = null; ParseData parseData = null; ParseText parseText = null; + while (values.hasNext()) { final Writable value = values.next().get(); // unwrap if (value instanceof Inlinks) { @@ -85,9 +87,30 @@ if (CrawlDatum.hasDbStatus(datum)) dbDatum = datum; else if (CrawlDatum.hasFetchStatus(datum)) { + // don't index unmodified (empty) pages - if (datum.getStatus() != CrawlDatum.STATUS_FETCH_NOTMODIFIED) + if (datum.getStatus() != CrawlDatum.STATUS_FETCH_NOTMODIFIED) { fetchDatum = datum; + + /** + * Check if we need to delete 404 NOT FOUND and 301 PERMANENT REDIRECT. + */ + if (delete) { + if (fetchDatum.getStatus() == CrawlDatum.STATUS_FETCH_GONE) { + System.out.println("Delete " + key); + reporter.incrCounter("IndexerStatus", "Documents deleted", 1); + output.collect(key, null); + continue; + } + if (fetchDatum.getStatus() == CrawlDatum.STATUS_FETCH_REDIR_PERM) { + System.out.println("Delete " + key); + reporter.incrCounter("IndexerStatus", "Perm redirects deleted", 1); + output.collect(key, null); + continue; + } + } + } + } else if (CrawlDatum.STATUS_LINKED == datum.getStatus() || CrawlDatum.STATUS_SIGNATURE == datum.getStatus() || CrawlDatum.STATUS_PARSE_META == datum.getStatus()) { @@ -162,7 +185,7 @@ doc.add("boost", Float.toString(boost)); reporter.incrCounter("IndexerStatus", "Documents added", 1); - +System.out.println("Index " + key); output.collect(key, doc); } Index: src/java/org/apache/nutch/indexer/NutchIndexWriter.java =================================================================== --- src/java/org/apache/nutch/indexer/NutchIndexWriter.java (revision 1170103) +++ src/java/org/apache/nutch/indexer/NutchIndexWriter.java (working copy) @@ -25,6 +25,8 @@ public void write(NutchDocument doc) throws IOException; + public void delete(String key) throws IOException; + public void close() throws IOException; } Index: src/java/org/apache/nutch/indexer/IndexerOutputFormat.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexerOutputFormat.java (revision 1170103) +++ src/java/org/apache/nutch/indexer/IndexerOutputFormat.java (working copy) @@ -51,7 +51,12 @@ public void write(Text key, NutchDocument doc) throws IOException { for (final NutchIndexWriter writer : writers) { - writer.write(doc); + // Delete doc == null + if (doc == null) { + writer.delete(key.toString()); + } else { + writer.write(doc); + } } } };