Index: lib/lucene-core-1.9.1.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: lib/hadoop-0.5.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: lib/lucene-misc-1.9.1.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: lib/hadoop-0.7.1.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lib/hadoop-0.7.1.jar ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Index: lib/lucene-core-2.0.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lib/lucene-core-2.0.0.jar ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Index: lib/lucene-misc-2.0.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lib/lucene-misc-2.0.0.jar ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Index: src/test/org/apache/nutch/fetcher/TestFetcher.java =================================================================== --- src/test/org/apache/nutch/fetcher/TestFetcher.java (revision 463985) +++ src/test/org/apache/nutch/fetcher/TestFetcher.java (working copy) @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDBTestUtil; import org.apache.nutch.crawl.Generator; import org.apache.nutch.crawl.Injector; @@ -108,7 +108,7 @@ READ: do { - UTF8 key=new UTF8(); + Text key=new Text(); Content value=new Content(); if(!reader.next(key, value)) break READ; String contentString=new String(value.getContent()); Index: src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java =================================================================== --- src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java (revision 0) +++ src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java (revision 0) @@ -0,0 +1,154 @@ +package org.apache.nutch.indexer; + +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.MD5Hash; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.DateTools.Resolution; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.nutch.analysis.NutchDocumentAnalyzer; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.util.NutchConfiguration; + +import junit.framework.TestCase; + +public class TestDeleteDuplicates extends TestCase { + Configuration conf; + FileSystem fs; + Path root; + Path index1; + Path index2; + + public void setUp() throws Exception { + conf = NutchConfiguration.create(); + conf.set("fs.default.name", "local"); + fs = FileSystem.get(conf); + root = new Path("dedup2-test-" + new Random().nextInt()); + // create test indexes + index1 = createIndex("index1", true, 1.0f, 10L); + index2 = createIndex("index2", false, 2.0f, 20L); + } + + private Path createIndex(String name, boolean hashDup, float inc, long time) throws Exception { + Path idx = new Path(root, name); + Path sub = new Path(idx, "part-0000"); + Directory dir = FSDirectory.getDirectory(sub.toString(), true); + IndexWriter writer = new IndexWriter(dir, new NutchDocumentAnalyzer(conf), true); + Document doc = makeDoc(name, + MD5Hash.digest("1").toString(), + "http://www.example.com/1", + 1.0f, time); + writer.addDocument(doc); + if (hashDup) { + doc = makeDoc(name, + MD5Hash.digest("1").toString(), + "http://www.example.com/2", + 1.0f + inc, time + 1); + } else { + doc = makeDoc(name, + MD5Hash.digest("2").toString(), + "http://www.example.com/1", + 1.0f + inc, time + 1); + } + writer.addDocument(doc); + writer.close(); + return idx; + } + + private Document makeDoc(String segment, String digest, String url, float boost, long time) { + Document doc = new Document(); + doc.add(new Field("segment", segment, Field.Store.YES, Field.Index.NO)); + doc.add(new Field("digest", digest, Field.Store.YES, Field.Index.NO)); + doc.add(new Field("url", url, Field.Store.YES, Field.Index.TOKENIZED)); + doc.setBoost(boost); + doc.add(new Field("boost", "" + boost, Field.Store.YES, Field.Index.NO)); + doc.add(new Field("tstamp", DateTools.timeToString(time, Resolution.MILLISECOND), Field.Store.YES, Field.Index.NO)); + return doc; + } + + public void tearDown() throws Exception { + fs.delete(root); + } + + public void testHashDuplicates() throws Exception { + DeleteDuplicates dedup = new DeleteDuplicates(conf); + dedup.dedup(new Path[]{index1}); + FsDirectory dir = new FsDirectory(fs, new Path(index1, "part-0000"), false, conf); + IndexReader reader = IndexReader.open(dir); + assertEquals("only one doc left", reader.numDocs(), 1); + for (int i = 0; i < reader.maxDoc(); i++) { + if (reader.isDeleted(i)) { + System.out.println("-doc " + i + " deleted"); + continue; + } + Document doc = reader.document(i); + // make sure we got the right one + assertEquals("check url", "http://www.example.com/2", doc.get("url")); + System.out.println(doc); + } + reader.close(); + } + + public void testUrlDuplicates() throws Exception { + DeleteDuplicates dedup = new DeleteDuplicates(conf); + dedup.dedup(new Path[]{index2}); + FsDirectory dir = new FsDirectory(fs, new Path(index2, "part-0000"), false, conf); + IndexReader reader = IndexReader.open(dir); + assertEquals("only one doc left", reader.numDocs(), 1); + MD5Hash hash = MD5Hash.digest("2"); + for (int i = 0; i < reader.maxDoc(); i++) { + if (reader.isDeleted(i)) { + System.out.println("-doc " + i + " deleted"); + continue; + } + Document doc = reader.document(i); + // make sure we got the right one + assertEquals("check hash", hash.toString(), doc.get("digest")); + System.out.println(doc); + } + reader.close(); + } + + public void testMixedDuplicates() throws Exception { + DeleteDuplicates dedup = new DeleteDuplicates(conf); + dedup.dedup(new Path[]{index1, index2}); + FsDirectory dir = new FsDirectory(fs, new Path(index1, "part-0000"), false, conf); + IndexReader reader = IndexReader.open(dir); + assertEquals("only one doc left", reader.numDocs(), 1); + for (int i = 0; i < reader.maxDoc(); i++) { + if (reader.isDeleted(i)) { + System.out.println("-doc " + i + " deleted"); + continue; + } + Document doc = reader.document(i); + // make sure we got the right one + assertEquals("check url", "http://www.example.com/2", doc.get("url")); + System.out.println(doc); + } + reader.close(); + dir = new FsDirectory(fs, new Path(index2, "part-0000"), false, conf); + reader = IndexReader.open(dir); + assertEquals("only one doc left", reader.numDocs(), 1); + MD5Hash hash = MD5Hash.digest("2"); + for (int i = 0; i < reader.maxDoc(); i++) { + if (reader.isDeleted(i)) { + System.out.println("-doc " + i + " deleted"); + continue; + } + Document doc = reader.document(i); + // make sure we got the right one + assertEquals("check hash", hash.toString(), doc.get("digest")); + System.out.println(doc); + } + reader.close(); + } + +} Index: src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java =================================================================== --- src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java (revision 463985) +++ src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java (working copy) @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MapFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.util.NutchConfiguration; import junit.framework.TestCase; @@ -61,12 +61,12 @@ cd1 = new CrawlDatum(); cd1.setFetchInterval(1.0f); cd1.setFetchTime(time); - cd1.getMetaData().put(new UTF8("name"), new UTF8("cd1")); - cd1.getMetaData().put(new UTF8("cd1"), new UTF8("cd1")); + cd1.getMetaData().put(new Text("name"), new Text("cd1")); + cd1.getMetaData().put(new Text("cd1"), new Text("cd1")); cd2 = new CrawlDatum(); cd2.setFetchInterval(1.0f); cd2.setFetchTime(time + 10000); - cd2.getMetaData().put(new UTF8("name"), new UTF8("cd2")); + cd2.getMetaData().put(new Text("name"), new Text("cd2")); cd3 = new CrawlDatum(); cd3.setFetchInterval(1.0f); cd3.setFetchTime(time + 10000); @@ -125,11 +125,11 @@ private void createCrawlDb(FileSystem fs, Path crawldb, TreeSet init, CrawlDatum cd) throws Exception { LOG.fine("* creating crawldb: " + crawldb); Path dir = new Path(crawldb, CrawlDatum.DB_DIR_NAME); - MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), UTF8.class, CrawlDatum.class); + MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), Text.class, CrawlDatum.class); Iterator it = init.iterator(); while (it.hasNext()) { String key = (String)it.next(); - writer.append(new UTF8(key), cd); + writer.append(new Text(key), cd); } writer.close(); } Index: src/test/org/apache/nutch/crawl/TestLinkDbMerger.java =================================================================== --- src/test/org/apache/nutch/crawl/TestLinkDbMerger.java (revision 463985) +++ src/test/org/apache/nutch/crawl/TestLinkDbMerger.java (working copy) @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MapFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.util.NutchConfiguration; import junit.framework.TestCase; @@ -122,7 +122,7 @@ String url = (String)it.next(); LOG.fine("url=" + url); String[] vals = (String[])expected.get(url); - Inlinks inlinks = reader.getInlinks(new UTF8(url)); + Inlinks inlinks = reader.getInlinks(new Text(url)); // may not be null assertNotNull(inlinks); ArrayList links = new ArrayList(); @@ -143,7 +143,7 @@ private void createLinkDb(FileSystem fs, Path linkdb, TreeMap init) throws Exception { LOG.fine("* creating linkdb: " + linkdb); Path dir = new Path(linkdb, LinkDb.CURRENT_NAME); - MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), UTF8.class, Inlinks.class); + MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), Text.class, Inlinks.class); Iterator it = init.keySet().iterator(); while (it.hasNext()) { String key = (String)it.next(); @@ -153,7 +153,7 @@ Inlink in = new Inlink(vals[i], vals[i]); inlinks.add(in); } - writer.append(new UTF8(key), inlinks); + writer.append(new Text(key), inlinks); } writer.close(); } Index: src/test/org/apache/nutch/crawl/TestGenerator.java =================================================================== --- src/test/org/apache/nutch/crawl/TestGenerator.java (revision 463985) +++ src/test/org/apache/nutch/crawl/TestGenerator.java (working copy) @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDBTestUtil.URLCrawlDatum; import junit.framework.TestCase; @@ -80,7 +80,7 @@ ArrayList list = new ArrayList(); for(int i=0;i<=100;i++){ - list.add(new CrawlDBTestUtil.URLCrawlDatum(new UTF8("http://aaa/" + pad(i)), + list.add(new CrawlDBTestUtil.URLCrawlDatum(new Text("http://aaa/" + pad(i)), new CrawlDatum(CrawlDatum.STATUS_DB_UNFETCHED, 1, i))); } @@ -105,7 +105,7 @@ READ: do { - UTF8 key=new UTF8(); + Text key=new Text(); CrawlDatum value=new CrawlDatum(); if(!reader.next(key, value)) break READ; l.add(new URLCrawlDatum(key, value)); Index: src/test/org/apache/nutch/crawl/TestMapWritable.java =================================================================== --- src/test/org/apache/nutch/crawl/TestMapWritable.java (revision 463985) +++ src/test/org/apache/nutch/crawl/TestMapWritable.java (working copy) @@ -25,7 +25,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.MapWritable; @@ -39,11 +39,11 @@ MapWritable map = new MapWritable(); assertTrue(map.isEmpty()); for (int i = 0; i < 100; i++) { - UTF8 key = new UTF8("" + i); + Text key = new Text("" + i); IntWritable value = new IntWritable(i); map.put(key, value); assertEquals(i + 1, map.size()); - assertTrue(map.containsKey(new UTF8("" + i))); + assertTrue(map.containsKey(new Text("" + i))); assertTrue(map.containsValue(new IntWritable(i))); map.remove(key); assertEquals(i, map.size()); @@ -64,14 +64,14 @@ map.clear(); assertTrue(map.isEmpty()); assertEquals(0, map.size()); - assertFalse(map.containsKey(new UTF8("" + 1))); + assertFalse(map.containsKey(new Text("" + 1))); } public void testWritable() throws Exception { MapWritable datum1 = new MapWritable(); for (int i = 0; i < 100; i++) { - datum1.put(new LongWritable(i), new UTF8("" + 1)); + datum1.put(new LongWritable(i), new Text("" + 1)); } assertEquals(100, datum1.size()); testWritable(datum1); @@ -86,7 +86,7 @@ CrawlDatum c = new CrawlDatum(CrawlDatum.STATUS_DB_FETCHED, 1f); c.setMetaData(new MapWritable()); for (int i = 0; i < 100; i++) { - c.getMetaData().put(new LongWritable(i), new UTF8("" + 1)); + c.getMetaData().put(new LongWritable(i), new Text("" + 1)); } testWritable(c); } @@ -94,10 +94,10 @@ public void testEquals() { MapWritable map1 = new MapWritable(); MapWritable map2 = new MapWritable(); - map1.put(new UTF8("key1"), new UTF8("val1")); - map1.put(new UTF8("key2"), new UTF8("val2")); - map2.put(new UTF8("key2"), new UTF8("val2")); - map2.put(new UTF8("key1"), new UTF8("val1")); + map1.put(new Text("key1"), new Text("val1")); + map1.put(new Text("key2"), new Text("val2")); + map2.put(new Text("key2"), new Text("val2")); + map2.put(new Text("key1"), new Text("val1")); assertTrue(map1.equals(map2)); } @@ -137,13 +137,13 @@ System.out.println("needed time for reading map's: " + needed); fs.delete(file); - // UTF8 + // Text System.out.println("start writing utf8's"); - writer = new SequenceFile.Writer(fs, file, IntWritable.class, UTF8.class); + writer = new SequenceFile.Writer(fs, file, IntWritable.class, Text.class); // write map start = System.currentTimeMillis(); key = new IntWritable(); - UTF8 value = new UTF8(); + Text value = new Text(); String s = "15726:15726"; for (int i = 0; i < 1000000; i++) { key.set(i); @@ -181,9 +181,9 @@ } public void testRecycling() throws Exception { - UTF8 value = new UTF8("value"); - UTF8 key1 = new UTF8("a"); - UTF8 key2 = new UTF8("b"); + Text value = new Text("value"); + Text key1 = new Text("a"); + Text key2 = new Text("b"); MapWritable writable = new MapWritable(); writable.put(key1, value); Index: src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java =================================================================== --- src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java (revision 463985) +++ src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java (working copy) @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MapFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.mortbay.http.HttpContext; import org.mortbay.http.SocketListener; import org.mortbay.http.handler.ResourceHandler; @@ -54,12 +54,12 @@ LOG.trace("* creating crawldb: " + crawldb); Path dir = new Path(crawldb, CrawlDatum.DB_DIR_NAME); MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000") - .toString(), UTF8.class, CrawlDatum.class); + .toString(), Text.class, CrawlDatum.class); Iterator it = init.iterator(); while (it.hasNext()) { URLCrawlDatum row = it.next(); LOG.info("adding:" + row.url.toString()); - writer.append(new UTF8(row.url), row.datum); + writer.append(new Text(row.url), row.datum); } writer.close(); } @@ -92,11 +92,11 @@ public static class URLCrawlDatum { - UTF8 url; + Text url; CrawlDatum datum; - public URLCrawlDatum(UTF8 url, CrawlDatum datum) { + public URLCrawlDatum(Text url, CrawlDatum datum) { this.url = url; this.datum = datum; } Index: src/test/org/apache/nutch/crawl/TestInjector.java =================================================================== --- src/test/org/apache/nutch/crawl/TestInjector.java (revision 463985) +++ src/test/org/apache/nutch/crawl/TestInjector.java (working copy) @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import junit.framework.TestCase; @@ -111,7 +111,7 @@ READ: do { - UTF8 key=new UTF8(); + Text key=new Text(); CrawlDatum value=new CrawlDatum(); if(!reader.next(key, value)) break READ; read.add(key.toString()); Index: src/java/org/apache/nutch/metadata/Metadata.java =================================================================== --- src/java/org/apache/nutch/metadata/Metadata.java (revision 463985) +++ src/java/org/apache/nutch/metadata/Metadata.java (working copy) @@ -34,7 +34,7 @@ import org.apache.commons.lang.StringUtils; // Hadoop imports -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -305,11 +305,11 @@ String[] values = null; String[] names = names(); for (int i=0; i 0) // write sort value class - UTF8.writeString(out, top[0].getSortValue().getClass().getName()); + Text.writeString(out, top[0].getSortValue().getClass().getName()); for (int i = 0; i < top.length; i++) { Hit h = top[i]; out.writeInt(h.getIndexDocNo()); // write indexDocNo h.getSortValue().write(out); // write sortValue - UTF8.writeString(out, h.getDedupValue()); // write dedupValue + Text.writeString(out, h.getDedupValue()); // write dedupValue } } @@ -85,7 +85,7 @@ Class sortClass = null; if (top.length > 0) { // read sort value class try { - sortClass = Class.forName(UTF8.readString(in)); + sortClass = Class.forName(Text.readString(in)); } catch (ClassNotFoundException e) { throw new IOException(e.toString()); } @@ -102,7 +102,7 @@ } sortValue.readFields(in); // read sortValue - String dedupValue = UTF8.readString(in); // read dedupValue + String dedupValue = Text.readString(in); // read dedupValue top[i] = new Hit(indexDocNo, sortValue, dedupValue); } Index: src/java/org/apache/nutch/searcher/FetchedSegments.java =================================================================== --- src/java/org/apache/nutch/searcher/FetchedSegments.java (revision 463985) +++ src/java/org/apache/nutch/searcher/FetchedSegments.java (working copy) @@ -53,7 +53,7 @@ this.conf = conf; } - public CrawlDatum getCrawlDatum(UTF8 url) throws IOException { + public CrawlDatum getCrawlDatum(Text url) throws IOException { synchronized (this) { if (crawl == null) crawl = getReaders(CrawlDatum.FETCH_DIR_NAME); @@ -61,7 +61,7 @@ return (CrawlDatum)getEntry(crawl, url, new CrawlDatum()); } - public byte[] getContent(UTF8 url) throws IOException { + public byte[] getContent(Text url) throws IOException { synchronized (this) { if (content == null) content = getReaders(Content.DIR_NAME); @@ -69,7 +69,7 @@ return ((Content)getEntry(content, url, new Content())).getContent(); } - public ParseData getParseData(UTF8 url) throws IOException { + public ParseData getParseData(Text url) throws IOException { synchronized (this) { if (parseData == null) parseData = getReaders(ParseData.DIR_NAME); @@ -77,7 +77,7 @@ return (ParseData)getEntry(parseData, url, new ParseData()); } - public ParseText getParseText(UTF8 url) throws IOException { + public ParseText getParseText(Text url) throws IOException { synchronized (this) { if (parseText == null) parseText = getReaders(ParseText.DIR_NAME); @@ -89,7 +89,7 @@ return MapFileOutputFormat.getReaders(fs, new Path(segmentDir, subDir), this.conf); } - private Writable getEntry(MapFile.Reader[] readers, UTF8 url, + private Writable getEntry(MapFile.Reader[] readers, Text url, Writable entry) throws IOException { return MapFileOutputFormat.getEntry(readers, PARTITIONER, url, entry); } @@ -212,8 +212,8 @@ return (Segment)segments.get(details.getValue("segment")); } - private UTF8 getUrl(HitDetails details) { - return new UTF8(details.getValue("url")); + private Text getUrl(HitDetails details) { + return new Text(details.getValue("url")); } public void close() throws IOException { Index: src/java/org/apache/nutch/searcher/IndexSearcher.java =================================================================== --- src/java/org/apache/nutch/searcher/IndexSearcher.java (revision 463985) +++ src/java/org/apache/nutch/searcher/IndexSearcher.java (working copy) @@ -152,7 +152,7 @@ } else if (raw instanceof Float) { sortValue = new FloatWritable(((Float)raw).floatValue()); } else if (raw instanceof String) { - sortValue = new UTF8((String)raw); + sortValue = new Text((String)raw); } else { throw new RuntimeException("Unknown sort value type!"); } Index: src/java/org/apache/nutch/searcher/LinkDbInlinks.java =================================================================== --- src/java/org/apache/nutch/searcher/LinkDbInlinks.java (revision 463985) +++ src/java/org/apache/nutch/searcher/LinkDbInlinks.java (working copy) @@ -12,7 +12,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -30,11 +30,11 @@ } public String[] getAnchors(HitDetails details) throws IOException { - return linkdb.getAnchors(new UTF8(details.getValue("url"))); + return linkdb.getAnchors(new Text(details.getValue("url"))); } public Inlinks getInlinks(HitDetails details) throws IOException { - return linkdb.getInlinks(new UTF8(details.getValue("url"))); + return linkdb.getInlinks(new Text(details.getValue("url"))); } public void close() throws IOException { Index: src/java/org/apache/nutch/searcher/Summary.java =================================================================== --- src/java/org/apache/nutch/searcher/Summary.java (revision 463985) +++ src/java/org/apache/nutch/searcher/Summary.java (working copy) @@ -23,7 +23,7 @@ import java.util.ArrayList; // Hadoop imports -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; // Nutch imports @@ -187,12 +187,12 @@ fragment = (Fragment) fragments.get(i); if (fragment.isHighlight()) { out.writeByte(HIGHLIGHT); - UTF8.writeString(out, fragment.getText()); + Text.writeString(out, fragment.getText()); } else if (fragment.isEllipsis()) { out.writeByte(ELLIPSIS); } else { out.writeByte(FRAGMENT); - UTF8.writeString(out, fragment.getText()); + Text.writeString(out, fragment.getText()); } } } @@ -204,11 +204,11 @@ for (int i=0; i + *
  • Phase 1 - remove URL duplicates:
    + * In this phase documents with the same URL + * are compared, and only the most recent document is retained - + * all other URL duplicates are scheduled for deletion.
    • + *
  • Phase 2 - remove content duplicates:
    + * In this phase documents with the same content hash are compared. If + * property "dedup.keep.highest.score" is set to true (default) then only + * the document with the highest score is retained. If this property is set + * to false, only the document with the shortest URL is retained - all other + * content duplicates are scheduled for deletion.
    • + *
  • Phase 3 - delete documents:
    + * In this phase documents scheduled for deletion are deleted from + * Lucene index(es).
    • + * + * + * @author Andrzej Bialecki + */ public class DeleteDuplicates extends ToolBase implements Mapper, Reducer, OutputFormat { private static final Log LOG = LogFactory.getLog(DeleteDuplicates.class); // Algorithm: // -// 1. map indexes -> <, > +// 1. map indexes -> > +// reduce, deleting all but most recent +// +// 2. map indexes -> > // partition by md5 -// reduce, deleting all but largest score w/ shortest url -// -// 2. map indexes -> <, > -// partition by url -// reduce, deleting all but most recent. -// -// Part 2 is not yet implemented, but the Indexer currently only indexes one -// URL per page, so this is not a critical problem. +// reduce, deleting all but with highest score (or shortest url). public static class IndexDoc implements WritableComparable { - private UTF8 index; // the segment index + private Text url = new Text(); + private int urlLen; + private float score; + private long time; + private MD5Hash hash = new MD5Hash(); + private Text index = new Text(); // the segment index private int doc; // within the index + private boolean keep = true; // keep or discard + public String toString() { + return "[url=" + url + ",score=" + score + ",time=" + time + + ",hash=" + hash + ",index=" + index + ",doc=" + doc + + ",keep=" + keep + "]"; + } + public void write(DataOutput out) throws IOException { + url.write(out); + out.writeFloat(score); + out.writeLong(time); + hash.write(out); index.write(out); out.writeInt(doc); + out.writeBoolean(keep); } public void readFields(DataInput in) throws IOException { - if (index == null) { - index = new UTF8(); - } + url.readFields(in); + urlLen = url.getLength(); + score = in.readFloat(); + time = in.readLong(); + hash.readFields(in); index.readFields(in); - this.doc = in.readInt(); + doc = in.readInt(); + keep = in.readBoolean(); } public int compareTo(Object o) { IndexDoc that = (IndexDoc)o; - int indexCompare = this.index.compareTo(that.index); - if (indexCompare != 0) { // prefer later indexes - return indexCompare; + if (this.keep != that.keep) { + return this.keep ? 1 : -1; + } else if (!this.hash.equals(that.hash)) { // order first by hash + return this.hash.compareTo(that.hash); + } else if (this.time != that.time) { // prefer more recent docs + return this.time > that.time ? 1 : -1 ; + } else if (this.urlLen != this.urlLen) { // prefer shorter urls + return this.urlLen - that.urlLen; } else { - return this.doc - that.doc; // prefer later docs + return this.score > that.score ? 1 : -1; } } public boolean equals(Object o) { IndexDoc that = (IndexDoc)o; - return this.index.equals(that.index) && this.doc == that.doc; + return this.keep == that.keep + && this.hash.equals(that.hash) + && this.time == that.time + && this.score == that.score + && this.urlLen == that.urlLen + && this.index.equals(that.index) + && this.doc == that.doc; } } - public static class HashScore implements WritableComparable { - private MD5Hash hash; - private float score; - private int urlLen; - - public void write(DataOutput out) throws IOException { - hash.write(out); - out.writeFloat(score); - out.writeInt(urlLen); - } - - public void readFields(DataInput in) throws IOException { - if (hash == null) { - hash = new MD5Hash(); - } - hash.readFields(in); - score = in.readFloat(); - urlLen = in.readInt(); - } - - public int compareTo(Object o) { - HashScore that = (HashScore)o; - if (!this.hash.equals(that.hash)) { // order first by hash - return this.hash.compareTo(that.hash); - } else if (this.score != that.score) { // prefer larger scores - return this.score < that.score ? 1 : -1 ; - } else { // prefer shorter urls - return this.urlLen - that.urlLen; - } - } - - public boolean equals(Object o) { - HashScore that = (HashScore)o; - return this.hash.equals(that.hash) - && this.score == that.score - && this.urlLen == that.urlLen; - } - } - public static class InputFormat extends InputFormatBase { private static final long INDEX_LENGTH = Integer.MAX_VALUE; @@ -145,94 +155,178 @@ return splits; } - /** Return each index as a split. */ - public RecordReader getRecordReader(final FileSystem fs, - final FileSplit split, - final JobConf job, - Reporter reporter) throws IOException { - final UTF8 index = new UTF8(split.getPath().toString()); - reporter.setStatus(index.toString()); - return new RecordReader() { + public class DDRecordReader implements RecordReader { - private IndexReader indexReader = - IndexReader.open(new FsDirectory(fs, split.getPath(), false, job)); + private IndexReader indexReader; + private int maxDoc; + private int doc; + private Text index; + + public DDRecordReader(FileSystem fs, FileSplit split, JobConf job, + Text index) throws IOException { + indexReader = IndexReader.open(new FsDirectory(fs, split.getPath(), false, job)); + maxDoc = indexReader.maxDoc(); + this.index = index; + } - { indexReader.undeleteAll(); } + public boolean next(Writable key, Writable value) + throws IOException { - private final int maxDoc = indexReader.maxDoc(); - private int doc; + // skip deleted documents + while (indexReader.isDeleted(doc) && doc < maxDoc) doc++; + if (doc >= maxDoc) + return false; - public boolean next(Writable key, Writable value) - throws IOException { + Document document = indexReader.document(doc); - if (doc >= maxDoc) - return false; + // fill in key + ((Text)key).set(document.get("url")); + // fill in value + IndexDoc indexDoc = (IndexDoc)value; + indexDoc.keep = true; + indexDoc.url.set(document.get("url")); + indexDoc.hash.setDigest(document.get("digest")); + indexDoc.score = Float.parseFloat(document.get("boost")); + try { + indexDoc.time = DateTools.stringToTime(document.get("tstamp")); + } catch (Exception e) { + // try to figure out the time from segment name + try { + String segname = document.get("segment"); + indexDoc.time = new SimpleDateFormat("yyyyMMddHHmmss").parse(segname).getTime(); + // make it unique + indexDoc.time += doc; + } catch (Exception e1) { + // use current time + indexDoc.time = System.currentTimeMillis(); + } + } + indexDoc.index = index; + indexDoc.doc = doc; - Document document = indexReader.document(doc); + doc++; - // fill in key - if (key instanceof UTF8) { - ((UTF8)key).set(document.get("url")); - } else { - HashScore hashScore = (HashScore)key; - if (hashScore.hash == null) { - hashScore.hash = new MD5Hash(); - } - hashScore.hash.setDigest(document.get("digest")); - hashScore.score = Float.parseFloat(document.get("boost")); - hashScore.urlLen = document.get("url").length(); - } + return true; + } - // fill in value - IndexDoc indexDoc = (IndexDoc)value; - if (indexDoc.index == null) { - indexDoc.index = new UTF8(); - } - indexDoc.index.set(index); - indexDoc.doc = doc; + public long getPos() throws IOException { + return maxDoc==0 ? 0 : (doc*INDEX_LENGTH)/maxDoc; + } - doc++; - - return true; - } - - public long getPos() throws IOException { - return maxDoc==0 ? 0 : (doc*INDEX_LENGTH)/maxDoc; - } - - public void close() throws IOException { - indexReader.close(); - } - }; + public void close() throws IOException { + indexReader.close(); + } + + public WritableComparable createKey() { + return new Text(); + } + + public Writable createValue() { + return new IndexDoc(); + } } + + /** Return each index as a split. */ + public RecordReader getRecordReader(final FileSystem fs, + final FileSplit split, + final JobConf job, + Reporter reporter) throws IOException { + final Text index = new Text(split.getPath().toString()); + reporter.setStatus(index.toString()); + return new DDRecordReader(fs, split, job, index); + } } - + public static class HashPartitioner implements Partitioner { public void configure(JobConf job) {} public void close() {} public int getPartition(WritableComparable key, Writable value, int numReduceTasks) { - int hashCode = ((HashScore)key).hash.hashCode(); + int hashCode = ((MD5Hash)key).hashCode(); return (hashCode & Integer.MAX_VALUE) % numReduceTasks; } } - public static class HashReducer implements Reducer { - private MD5Hash prevHash = new MD5Hash(); + public static class UrlsReducer implements Reducer { + public void configure(JobConf job) {} + public void close() {} + public void reduce(WritableComparable key, Iterator values, + OutputCollector output, Reporter reporter) throws IOException { + IndexDoc latest = null; + while (values.hasNext()) { + IndexDoc value = (IndexDoc)values.next(); + if (latest == null) { + latest = value; + continue; + } + if (value.time > latest.time) { + // discard current and use more recent + latest.keep = false; + LOG.debug("-discard " + latest + ", keep " + value); + output.collect(latest.hash, latest); + latest = value; + } else { + // discard + value.keep = false; + LOG.debug("-discard " + value + ", keep " + latest); + output.collect(value.hash, value); + } + + } + // keep the latest + latest.keep = true; + output.collect(latest.hash, latest); + + } + } + + public static class HashReducer implements Reducer { + boolean byScore; + + public void configure(JobConf job) { + byScore = job.getBoolean("dedup.keep.highest.score", true); + } + + public void close() {} + public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { - MD5Hash hash = ((HashScore)key).hash; + IndexDoc highest = null; while (values.hasNext()) { - Writable value = (Writable)values.next(); - if (hash.equals(prevHash)) { // collect all but first - output.collect(key, value); + IndexDoc value = (IndexDoc)values.next(); + // skip already deleted + if (!value.keep) { + LOG.debug("-discard " + value + " (already marked)"); + output.collect(value.url, value); + continue; + } + if (highest == null) { + highest = value; + continue; + } + if (byScore) { + if (value.score > highest.score) { + highest.keep = false; + LOG.debug("-discard " + highest + ", keep " + value); + output.collect(highest.url, highest); // delete highest + highest = value; + } } else { - prevHash.set(hash); + if (value.urlLen < highest.urlLen) { + highest.keep = false; + LOG.debug("-discard " + highest + ", keep " + value); + output.collect(highest.url, highest); // delete highest + highest = value; + } } } + LOG.debug("-keep " + highest); + // no need to add this - in phase 2 we only process docs to delete them + // highest.keep = true; + // output.collect(key, highest); } } @@ -240,8 +334,12 @@ public void configure(JobConf job) { setConf(job); + } + + public void setConf(Configuration conf) { + super.setConf(conf); try { - fs = FileSystem.get(job); + fs = FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } @@ -254,6 +352,9 @@ OutputCollector output, Reporter reporter) throws IOException { IndexDoc indexDoc = (IndexDoc)value; + // don't delete these + if (indexDoc.keep) return; + // delete all others output.collect(indexDoc.index, new IntWritable(indexDoc.doc)); } @@ -265,7 +366,9 @@ IndexReader reader = IndexReader.open(new FsDirectory(fs, index, false, getConf())); try { while (values.hasNext()) { - reader.deleteDocument(((IntWritable)values.next()).get()); + IntWritable value = (IntWritable)values.next(); + LOG.debug("-delete " + index + " doc=" + value); + reader.deleteDocument(value.get()); } } finally { reader.close(); @@ -301,8 +404,8 @@ if (LOG.isInfoEnabled()) { LOG.info("Dedup: starting"); } - Path hashDir = - new Path("dedup-hash-"+ + Path outDir1 = + new Path("dedup-urls-"+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); JobConf job = new NutchJob(getConf()); @@ -313,44 +416,67 @@ } job.addInputPath(indexDirs[i]); } - job.setJobName("dedup phase 1"); + job.setJobName("dedup 1: urls by time"); - job.setInputKeyClass(HashScore.class); - job.setInputValueClass(IndexDoc.class); job.setInputFormat(InputFormat.class); - job.setBoolean("mapred.speculative.execution", false); + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(IndexDoc.class); + job.setReducerClass(UrlsReducer.class); + job.setOutputPath(outDir1); + + job.setOutputKeyClass(MD5Hash.class); + job.setOutputValueClass(IndexDoc.class); + job.setOutputFormat(SequenceFileOutputFormat.class); + + JobClient.runJob(job); + + Path outDir2 = + new Path("dedup-hash-"+ + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); + job = new NutchJob(getConf()); + job.setJobName("dedup 2: content by hash"); + + job.addInputPath(outDir1); + //job.setInputKeyClass(MD5Hash.class); + //job.setInputValueClass(IndexDoc.class); + job.setInputFormat(SequenceFileInputFormat.class); + job.setMapOutputKeyClass(MD5Hash.class); + job.setMapOutputValueClass(IndexDoc.class); job.setPartitionerClass(HashPartitioner.class); + job.setSpeculativeExecution(false); + job.setReducerClass(HashReducer.class); + job.setOutputPath(outDir2); - job.setOutputPath(hashDir); - - job.setOutputKeyClass(HashScore.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(IndexDoc.class); job.setOutputFormat(SequenceFileOutputFormat.class); JobClient.runJob(job); + // remove outDir1 - no longer needed + fs.delete(outDir1); + job = new NutchJob(getConf()); - job.setJobName("dedup phase 2"); + job.setJobName("dedup 3: delete from index(es)"); - job.addInputPath(hashDir); - + job.addInputPath(outDir2); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(HashScore.class); - job.setInputValueClass(IndexDoc.class); + //job.setInputKeyClass(Text.class); + //job.setInputValueClass(IndexDoc.class); job.setInt("io.file.buffer.size", 4096); job.setMapperClass(DeleteDuplicates.class); job.setReducerClass(DeleteDuplicates.class); job.setOutputFormat(DeleteDuplicates.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); JobClient.runJob(job); - new JobClient(getConf()).getFs().delete(hashDir); + fs.delete(outDir2); if (LOG.isInfoEnabled()) { LOG.info("Dedup: done"); } } @@ -363,7 +489,7 @@ public int run(String[] args) throws Exception { if (args.length < 1) { - System.err.println("Usage: ..."); + System.err.println("Usage: DeleteDuplicates ..."); return -1; } Index: src/java/org/apache/nutch/indexer/IndexSorter.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexSorter.java (revision 463985) +++ src/java/org/apache/nutch/indexer/IndexSorter.java (working copy) @@ -27,12 +27,12 @@ import org.apache.lucene.search.*; import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.ToolBase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; /** Sort a Nutch index by page score. Higher scoring documents are assigned * smaller document numbers. */ Index: src/java/org/apache/nutch/indexer/IndexMerger.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexMerger.java (revision 463985) +++ src/java/org/apache/nutch/indexer/IndexMerger.java (working copy) @@ -24,11 +24,11 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.hadoop.conf.*; import org.apache.nutch.util.LogUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.ToolBase; import org.apache.lucene.store.Directory; import org.apache.lucene.index.IndexWriter; Index: src/java/org/apache/nutch/indexer/IndexingFilter.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexingFilter.java (revision 463985) +++ src/java/org/apache/nutch/indexer/IndexingFilter.java (working copy) @@ -21,7 +21,7 @@ // Hadoop imports import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; // Nutch imports import org.apache.nutch.parse.Parse; @@ -50,6 +50,6 @@ * @return modified (or a new) document instance * @throws IndexingException */ - Document filter(Document doc, Parse parse, UTF8 url, CrawlDatum datum, Inlinks inlinks) + Document filter(Document doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException; } Index: src/java/org/apache/nutch/indexer/Indexer.java =================================================================== --- src/java/org/apache/nutch/indexer/Indexer.java (revision 463985) +++ src/java/org/apache/nutch/indexer/Indexer.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.nutch.parse.*; import org.apache.nutch.analysis.*; @@ -37,7 +38,6 @@ import org.apache.nutch.util.LogUtil; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.Inlinks; @@ -74,6 +74,11 @@ } return super.next(key, (Writable)wrapper.get()); } + + // override the default - we want ObjectWritable-s here + public Writable createValue() { + return new ObjectWritable(); + } }; } } @@ -233,7 +238,7 @@ Parse parse = new ParseImpl(parseText, parseData); try { // run indexing filters - doc = this.filters.filter(doc, parse, (UTF8)key, fetchDatum, inlinks); + doc = this.filters.filter(doc, parse, (Text)key, fetchDatum, inlinks); } catch (IndexingException e) { if (LOG.isWarnEnabled()) { LOG.warn("Error indexing "+key+": "+e); } return; @@ -242,7 +247,7 @@ float boost = 1.0f; // run scoring filters try { - boost = this.scfilters.indexerScore((UTF8)key, doc, dbDatum, + boost = this.scfilters.indexerScore((Text)key, doc, dbDatum, fetchDatum, parse, inlinks, boost); } catch (ScoringFilterException e) { if (LOG.isWarnEnabled()) { @@ -283,15 +288,15 @@ job.addInputPath(new Path(linkDb, LinkDb.CURRENT_NAME)); job.setInputFormat(InputFormat.class); - job.setInputKeyClass(UTF8.class); - job.setInputValueClass(ObjectWritable.class); + //job.setInputKeyClass(Text.class); + //job.setInputValueClass(ObjectWritable.class); //job.setCombinerClass(Indexer.class); job.setReducerClass(Indexer.class); job.setOutputPath(indexDir); job.setOutputFormat(OutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(ObjectWritable.class); JobClient.runJob(job); Index: src/java/org/apache/nutch/indexer/IndexingFilters.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexingFilters.java (revision 463985) +++ src/java/org/apache/nutch/indexer/IndexingFilters.java (working copy) @@ -29,7 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.Inlinks; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; /** Creates and caches {@link IndexingFilter} implementing plugins.*/ public class IndexingFilters { @@ -66,7 +66,7 @@ } /** Run all defined filters. */ - public Document filter(Document doc, Parse parse, UTF8 url, CrawlDatum datum, Inlinks inlinks) + public Document filter(Document doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { for (int i = 0; i < this.indexingFilters.length; i++) { doc = this.indexingFilters[i].filter(doc, parse, url, datum, inlinks); Index: src/java/org/apache/nutch/tools/compat/CrawlDbConverter.java =================================================================== --- src/java/org/apache/nutch/tools/compat/CrawlDbConverter.java (revision 0) +++ src/java/org/apache/nutch/tools/compat/CrawlDbConverter.java (revision 0) @@ -0,0 +1,141 @@ +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.tools.compat; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapFileOutputFormat; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.CrawlDb; +import org.apache.nutch.crawl.MapWritable; +import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.NutchJob; + +/** + * This tool converts CrawlDb created in old <UTF8, CrawlDatum> format + * (Nutch versions < 0.9.0) to the new <Text, CrawlDatum> format. + * Optionally {@link org.apache.nutch.crawl.CrawlDatum#metaData} can be converted + * too from using UTF8 keys to using Text keys. + * + * @author Andrzej Bialecki + */ +public class CrawlDbConverter extends ToolBase implements Mapper { + private static final Log LOG = LogFactory.getLog(CrawlDbConverter.class); + + private static final String CONVERT_META_KEY = "db.converter.with.metadata"; + + private boolean withMetadata; + private Text newKey; + + public void configure(JobConf job) { + setConf(job); + withMetadata = job.getBoolean(CONVERT_META_KEY, false); + newKey = new Text(); + } + + public void map(WritableComparable key, Writable value, OutputCollector output, + Reporter reporter) throws IOException { + newKey.set(key.toString()); + if (withMetadata) { + CrawlDatum datum = (CrawlDatum)value; + MapWritable meta = datum.getMetaData(); + if (meta.size() > 0) { + MapWritable newMeta = new MapWritable(); + Iterator it = meta.keySet().iterator(); + while (it.hasNext()) { + WritableComparable k = (WritableComparable)it.next(); + Writable v = meta.get(k); + if (k instanceof UTF8) { + Text t = new Text(k.toString()); + k = t; + } + newMeta.put(k, v); + } + datum.setMetaData(newMeta); + } + } + output.collect(newKey, value); + } + + public void close() throws IOException { + } + + /** + * @param args + */ + public static void main(String[] args) throws Exception { + int res = new CrawlDbConverter().doMain(NutchConfiguration.create(), args); + } + + public int run(String[] args) throws Exception { + if (args.length == 0) { + System.err.println("Usage: CrawlDbConverter [-withMetadata]"); + System.err.println("\toldDb\tname of the crawldb that uses UTF8 class."); + System.err.println("\tnewDb\tname of the crawldb that will use Text class."); + System.err.println("\twithMetadata\tconvert also all metadata keys using UTF8 to Text."); + return -1; + } + JobConf job = new NutchJob(getConf()); + FileSystem fs = FileSystem.get(getConf()); + Path oldDb = new Path(args[0], CrawlDatum.DB_DIR_NAME); + Path newDb = + new Path(oldDb, + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); + if (!fs.exists(oldDb)) { + LOG.fatal("Old db doesn't exist in '" + args[0] + "'"); + return -1; + } + boolean withMetadata = false; + if (args.length > 2 && args[2].equalsIgnoreCase("-withMetadata")) + withMetadata = true; + + job.setBoolean(CONVERT_META_KEY, withMetadata); + job.setInputPath(oldDb); + job.setInputFormat(SequenceFileInputFormat.class); + job.setMapperClass(CrawlDbConverter.class); + job.setOutputFormat(MapFileOutputFormat.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(CrawlDatum.class); + job.setOutputPath(newDb); + try { + JobClient.runJob(job); + CrawlDb.install(job, new Path(args[1])); + return 0; + } catch (Exception e) { + LOG.fatal("Error: " + StringUtils.stringifyException(e)); + return -1; + } + } +} Property changes on: src/java/org/apache/nutch/tools/compat/CrawlDbConverter.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/nutch/tools/DmozParser.java =================================================================== --- src/java/org/apache/nutch/tools/DmozParser.java (revision 463985) +++ src/java/org/apache/nutch/tools/DmozParser.java (working copy) @@ -291,8 +291,8 @@ if (LOG.isInfoEnabled()) { LOG.info("skew = " + rp.hashSkew); } // - // Open filtered text stream. The UTF8Filter makes sure that - // only appropriate XML-approved UTF8 characters are received. + // Open filtered text stream. The TextFilter makes sure that + // only appropriate XML-approved Text characters are received. // Any non-conforming characters are silently skipped. // XMLCharFilter in = new XMLCharFilter(new BufferedReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(dmozFile)), "UTF-8"))); Index: src/java/org/apache/nutch/protocol/Protocol.java =================================================================== --- src/java/org/apache/nutch/protocol/Protocol.java (revision 463985) +++ src/java/org/apache/nutch/protocol/Protocol.java (working copy) @@ -18,7 +18,7 @@ // Hadoop imports import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; // Nutch imports import org.apache.nutch.crawl.CrawlDatum; @@ -32,5 +32,5 @@ /** Returns the {@link Content} for a fetchlist entry. */ - ProtocolOutput getProtocolOutput(UTF8 url, CrawlDatum datum); + ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum); } Index: src/java/org/apache/nutch/protocol/Content.java =================================================================== --- src/java/org/apache/nutch/protocol/Content.java (revision 463985) +++ src/java/org/apache/nutch/protocol/Content.java (working copy) @@ -32,7 +32,7 @@ public static final String DIR_NAME = "content"; - private final static byte VERSION = 1; + private final static byte VERSION = 2; private byte version; private String url; @@ -64,31 +64,54 @@ protected final void readFieldsCompressed(DataInput in) throws IOException { version = in.readByte(); - if (version > VERSION) - throw new VersionMismatchException(VERSION, version); + metadata = new Metadata(); + switch (version) { + case 0: + case 1: + url = UTF8.readString(in); // read url + base = UTF8.readString(in); // read base - url = UTF8.readString(in); // read url - base = UTF8.readString(in); // read base + content = new byte[in.readInt()]; // read content + in.readFully(content); - content = new byte[in.readInt()]; // read content - in.readFully(content); + contentType = UTF8.readString(in); // read contentType + // reconstruct metadata + int keySize = in.readInt(); + String key; + for (int i = 0; i < keySize; i++) { + key = UTF8.readString(in); + int valueSize = in.readInt(); + for (int j = 0; j < valueSize; j++) { + metadata.add(key, UTF8.readString(in)); + } + } + break; + case VERSION: + url = Text.readString(in); // read url + base = Text.readString(in); // read base - contentType = UTF8.readString(in); // read contentType + content = new byte[in.readInt()]; // read content + in.readFully(content); - metadata = new Metadata(); - metadata.readFields(in); // read meta data + contentType = Text.readString(in); // read contentType + metadata.readFields(in); // read meta data + break; + default: + throw new VersionMismatchException(VERSION, version); + } + } protected final void writeCompressed(DataOutput out) throws IOException { - out.writeByte(version); + out.writeByte(VERSION); - UTF8.writeString(out, url); // write url - UTF8.writeString(out, base); // write base + Text.writeString(out, url); // write url + Text.writeString(out, base); // write base out.writeInt(content.length); // write content out.write(content); - UTF8.writeString(out, contentType); // write contentType + Text.writeString(out, contentType); // write contentType metadata.write(out); // write metadata } @@ -171,6 +194,7 @@ ensureInflated(); StringBuffer buffer = new StringBuffer(); + buffer.append("Version: " + version + "\n" ); buffer.append("url: " + url + "\n" ); buffer.append("base: " + base + "\n" ); buffer.append("contentType: " + contentType + "\n" ); Index: src/java/org/apache/nutch/segment/SegmentMerger.java =================================================================== --- src/java/org/apache/nutch/segment/SegmentMerger.java (revision 463985) +++ src/java/org/apache/nutch/segment/SegmentMerger.java (working copy) @@ -89,10 +89,10 @@ public class SegmentMerger extends Configured implements Mapper, Reducer { private static final Log LOG = LogFactory.getLog(SegmentMerger.class); - private static final UTF8 SEGMENT_PART_KEY = new UTF8("_PaRt_"); - private static final UTF8 SEGMENT_NAME_KEY = new UTF8("_NaMe_"); + private static final Text SEGMENT_PART_KEY = new Text("_PaRt_"); + private static final Text SEGMENT_NAME_KEY = new Text("_NaMe_"); private static final String nameMarker = SEGMENT_NAME_KEY.toString(); - private static final UTF8 SEGMENT_SLICE_KEY = new UTF8("_SlIcE_"); + private static final Text SEGMENT_SLICE_KEY = new Text("_SlIcE_"); private static final String sliceMarker = SEGMENT_SLICE_KEY.toString(); private URLFilters filters = null; @@ -140,8 +140,8 @@ Object o = wrapper.get(); if (o instanceof CrawlDatum) { // record which part of segment this comes from - ((CrawlDatum)o).getMetaData().put(SEGMENT_PART_KEY, new UTF8(part)); - ((CrawlDatum)o).getMetaData().put(SEGMENT_NAME_KEY, new UTF8(segment)); + ((CrawlDatum)o).getMetaData().put(SEGMENT_PART_KEY, new Text(part)); + ((CrawlDatum)o).getMetaData().put(SEGMENT_NAME_KEY, new Text(segment)); } else if (o instanceof Content) { if (((Content)o).getMetadata() == null) { ((Content)o).setMetadata(new Metadata()); @@ -162,6 +162,10 @@ } return res; } + + public Writable createValue() { + return new ObjectWritable(); + } }; } } @@ -186,12 +190,12 @@ String slice = null; if (o instanceof CrawlDatum) { // check which output dir it should go into - UTF8 part = (UTF8)((CrawlDatum)o).getMetaData().get(SEGMENT_PART_KEY); + Text part = (Text)((CrawlDatum)o).getMetaData().get(SEGMENT_PART_KEY); ((CrawlDatum)o).getMetaData().remove(SEGMENT_PART_KEY); ((CrawlDatum)o).getMetaData().remove(SEGMENT_NAME_KEY); if (part == null) throw new IOException("Null segment part, key=" + key); - UTF8 uSlice = (UTF8)((CrawlDatum)o).getMetaData().get(SEGMENT_SLICE_KEY); + Text uSlice = (Text)((CrawlDatum)o).getMetaData().get(SEGMENT_SLICE_KEY); ((CrawlDatum)o).getMetaData().remove(SEGMENT_SLICE_KEY); if (uSlice != null) slice = uSlice.toString(); String partString = part.toString(); @@ -267,7 +271,7 @@ } else { wname = new Path(new Path(new Path(job.getOutputPath(), segmentName + "-" + slice), dirName), name); } - res = new SequenceFile.Writer(fs, wname, UTF8.class, CrawlDatum.class); + res = new SequenceFile.Writer(fs, job, wname, Text.class, CrawlDatum.class); sliceWriters.put(slice + dirName, res); return res; } @@ -283,7 +287,7 @@ } else { wname = new Path(new Path(new Path(job.getOutputPath(), segmentName + "-" + slice), dirName), name); } - res = new MapFile.Writer(fs, wname.toString(), UTF8.class, clazz); + res = new MapFile.Writer(fs, wname.toString(), Text.class, clazz); sliceWriters.put(slice + dirName, res); return res; } @@ -332,10 +336,17 @@ } } + private Text newKey = new Text(); + public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { + // convert on the fly from the old format + if (key instanceof UTF8) { + newKey.set(key.toString()); + key = newKey; + } if (filters != null) { try { - if (filters.filter(((UTF8)key).toString()) == null) { + if (filters.filter(((Text)key).toString()) == null) { return; } } catch (Exception e) { @@ -373,10 +384,10 @@ if (o instanceof CrawlDatum) { CrawlDatum val = (CrawlDatum)o; // check which output dir it belongs to - UTF8 part = (UTF8)val.getMetaData().get(SEGMENT_PART_KEY); + Text part = (Text)val.getMetaData().get(SEGMENT_PART_KEY); if (part == null) throw new IOException("Null segment part, key=" + key); - UTF8 uName = (UTF8)val.getMetaData().get(SEGMENT_NAME_KEY); + Text uName = (Text)val.getMetaData().get(SEGMENT_NAME_KEY); if (uName == null) throw new IOException("Null segment name, key=" + key); String name = uName.toString(); @@ -470,10 +481,10 @@ } } curCount++; - UTF8 sliceName = null; + Text sliceName = null; ObjectWritable wrapper = new ObjectWritable(); if (sliceSize > 0) { - sliceName = new UTF8(String.valueOf(curCount / sliceSize)); + sliceName = new Text(String.valueOf(curCount / sliceSize)); } // now output the latest values if (lastG != null) { @@ -613,12 +624,10 @@ } } job.setInputFormat(ObjectInputFormat.class); - job.setInputKeyClass(UTF8.class); - job.setInputValueClass(ObjectWritable.class); job.setMapperClass(SegmentMerger.class); job.setReducerClass(SegmentMerger.class); job.setOutputPath(out); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(ObjectWritable.class); job.setOutputFormat(SegmentOutputFormat.class); Index: src/java/org/apache/nutch/segment/SegmentReader.java =================================================================== --- src/java/org/apache/nutch/segment/SegmentReader.java (revision 463985) +++ src/java/org/apache/nutch/segment/SegmentReader.java (working copy) @@ -65,10 +65,28 @@ } return super.next(key, (Writable) wrapper.get()); } + + public Writable createValue() { + return new ObjectWritable(); + } }; } } + + public static class InputCompatMapper extends MapReduceBase implements Mapper { + private Text newKey = new Text(); + public void map(WritableComparable key, Writable value, OutputCollector collector, Reporter reporter) throws IOException { + // convert on the fly from old formats with UTF8 keys + if (key instanceof UTF8) { + newKey.set(key.toString()); + key = newKey; + } + collector.collect(key, value); + } + + } + /** Implements a text output format */ public static class TextOutputFormat extends org.apache.hadoop.mapred.OutputFormatBase { public RecordWriter getRecordWriter(final FileSystem fs, JobConf job, String name, Progressable progress) throws IOException { @@ -180,9 +198,7 @@ if (pt) job.addInputPath(new Path(segment, ParseText.DIR_NAME)); job.setInputFormat(InputFormat.class); - job.setInputKeyClass(UTF8.class); - job.setInputValueClass(ObjectWritable.class); - + job.setMapperClass(InputCompatMapper.class); job.setReducerClass(SegmentReader.class); Path tempDir = new Path("/tmp/segread-" + new java.util.Random().nextInt()); @@ -190,7 +206,7 @@ job.setOutputPath(tempDir); job.setOutputFormat(TextOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(ObjectWritable.class); JobClient.runJob(job); @@ -255,7 +271,7 @@ {"pt", "ParseText::\n"} }; - public void get(final Path segment, final UTF8 key, Writer writer, + public void get(final Path segment, final Text key, Writer writer, final Map results) throws Exception { if (LOG.isInfoEnabled()) { LOG.info("SegmentReader: get '" + key + "'"); } ArrayList threads = new ArrayList(); @@ -346,12 +362,12 @@ } } - private List getMapRecords(Path dir, UTF8 key) throws Exception { + private List getMapRecords(Path dir, Text key) throws Exception { MapFile.Reader[] readers = MapFileOutputFormat.getReaders(fs, dir, getConf()); ArrayList res = new ArrayList(); Class keyClass = readers[0].getKeyClass(); Class valueClass = readers[0].getValueClass(); - if (!keyClass.getName().equals("org.apache.hadoop.io.UTF8")) + if (!keyClass.getName().equals("org.apache.hadoop.io.Text")) throw new IOException("Incompatible key (" + keyClass.getName() + ")"); Writable value = (Writable)valueClass.newInstance(); // we don't know the partitioning schema @@ -363,12 +379,12 @@ return res; } - private List getSeqRecords(Path dir, UTF8 key) throws Exception { + private List getSeqRecords(Path dir, Text key) throws Exception { SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(getConf(), dir); ArrayList res = new ArrayList(); Class keyClass = readers[0].getKeyClass(); Class valueClass = readers[0].getValueClass(); - if (!keyClass.getName().equals("org.apache.hadoop.io.UTF8")) + if (!keyClass.getName().equals("org.apache.hadoop.io.Text")) throw new IOException("Incompatible key (" + keyClass.getName() + ")"); Writable aKey = (Writable)keyClass.newInstance(); Writable value = (Writable)valueClass.newInstance(); @@ -423,7 +439,7 @@ public void getStats(Path segment, final SegmentReaderStats stats) throws Exception { SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(getConf(), new Path(segment, CrawlDatum.GENERATE_DIR_NAME)); long cnt = 0L; - UTF8 key = new UTF8(); + Text key = new Text(); for (int i = 0; i < readers.length; i++) { while (readers[i].next(key)) cnt++; readers[i].close(); @@ -566,7 +582,7 @@ usage(); return; } - segmentReader.get(new Path(input), new UTF8(key), new OutputStreamWriter(System.out, "UTF-8"), new HashMap()); + segmentReader.get(new Path(input), new Text(key), new OutputStreamWriter(System.out, "UTF-8"), new HashMap()); return; default: System.err.println("Invalid operation: " + args[0]); Index: src/java/org/apache/nutch/scoring/ScoringFilter.java =================================================================== --- src/java/org/apache/nutch/scoring/ScoringFilter.java (revision 463985) +++ src/java/org/apache/nutch/scoring/ScoringFilter.java (working copy) @@ -18,7 +18,7 @@ import java.util.List; import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.lucene.document.Document; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.Inlinks; @@ -49,7 +49,7 @@ * @param datum new datum. Filters will modify it in-place. * @throws ScoringFilterException */ - public void injectedScore(UTF8 url, CrawlDatum datum) throws ScoringFilterException; + public void injectedScore(Text url, CrawlDatum datum) throws ScoringFilterException; /** * Set an initial score for newly discovered pages. Note: newly discovered pages @@ -60,7 +60,7 @@ * @param datum new datum. Filters will modify it in-place. * @throws ScoringFilterException */ - public void initialScore(UTF8 url, CrawlDatum datum) throws ScoringFilterException; + public void initialScore(Text url, CrawlDatum datum) throws ScoringFilterException; /** * This method prepares a sort value for the purpose of sorting and @@ -69,7 +69,7 @@ * @param datum page's datum, should not be modified * @param initSort initial sort value, or a value from previous filters in chain */ - public float generatorSortValue(UTF8 url, CrawlDatum datum, float initSort) throws ScoringFilterException; + public float generatorSortValue(Text url, CrawlDatum datum, float initSort) throws ScoringFilterException; /** * This method takes all relevant score information from the current datum @@ -82,7 +82,7 @@ * @param content instance of content. Implementations may modify this * in-place, primarily by setting some metadata properties. */ - public void passScoreBeforeParsing(UTF8 url, CrawlDatum datum, Content content) throws ScoringFilterException; + public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content) throws ScoringFilterException; /** * Currently a part of score distribution is performed using only data coming @@ -93,7 +93,7 @@ * @param parse target instance to copy the score information to. Implementations * may modify this in-place, primarily by setting some metadata properties. */ - public void passScoreAfterParsing(UTF8 url, Content content, Parse parse) throws ScoringFilterException; + public void passScoreAfterParsing(Text url, Content content, Parse parse) throws ScoringFilterException; /** * Distribute score value from the current page to all its outlinked pages. @@ -116,7 +116,7 @@ * be null if not needed. * @throws ScoringFilterException */ - public CrawlDatum distributeScoreToOutlink(UTF8 fromUrl, UTF8 toUrl, + public CrawlDatum distributeScoreToOutlink(Text fromUrl, Text toUrl, ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount, int validCount) throws ScoringFilterException; @@ -136,7 +136,7 @@ * links pointing to this page, found in the current update batch. * @throws ScoringFilterException */ - public void updateDbScore(UTF8 url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException; + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException; /** * This method calculates a Lucene document boost. @@ -156,6 +156,6 @@ * other scoring strategies by modifying Lucene document directly. * @throws ScoringFilterException */ - public float indexerScore(UTF8 url, Document doc, CrawlDatum dbDatum, + public float indexerScore(Text url, Document doc, CrawlDatum dbDatum, CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) throws ScoringFilterException; } Index: src/java/org/apache/nutch/scoring/ScoringFilters.java =================================================================== --- src/java/org/apache/nutch/scoring/ScoringFilters.java (revision 463985) +++ src/java/org/apache/nutch/scoring/ScoringFilters.java (working copy) @@ -32,7 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; /** * Creates and caches {@link ScoringFilter} implementing plugins. @@ -85,7 +85,7 @@ } /** Calculate a sort value for Generate. */ - public float generatorSortValue(UTF8 url, CrawlDatum datum, float initSort) throws ScoringFilterException { + public float generatorSortValue(Text url, CrawlDatum datum, float initSort) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { initSort = this.filters[i].generatorSortValue(url, datum, initSort); } @@ -93,46 +93,46 @@ } /** Calculate a new initial score, used when adding newly discovered pages. */ - public void initialScore(UTF8 url, CrawlDatum datum) throws ScoringFilterException { + public void initialScore(Text url, CrawlDatum datum) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { this.filters[i].initialScore(url, datum); } } /** Calculate a new initial score, used when injecting new pages. */ - public void injectedScore(UTF8 url, CrawlDatum datum) throws ScoringFilterException { + public void injectedScore(Text url, CrawlDatum datum) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { this.filters[i].injectedScore(url, datum); } } /** Calculate updated page score during CrawlDb.update(). */ - public void updateDbScore(UTF8 url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException { + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { this.filters[i].updateDbScore(url, old, datum, inlinked); } } - public void passScoreBeforeParsing(UTF8 url, CrawlDatum datum, Content content) throws ScoringFilterException { + public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { this.filters[i].passScoreBeforeParsing(url, datum, content); } } - public void passScoreAfterParsing(UTF8 url, Content content, Parse parse) throws ScoringFilterException { + public void passScoreAfterParsing(Text url, Content content, Parse parse) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { this.filters[i].passScoreAfterParsing(url, content, parse); } } - public CrawlDatum distributeScoreToOutlink(UTF8 fromUrl, UTF8 toUrl, ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount, int validCount) throws ScoringFilterException { + public CrawlDatum distributeScoreToOutlink(Text fromUrl, Text toUrl, ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount, int validCount) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { adjust = this.filters[i].distributeScoreToOutlink(fromUrl, toUrl, parseData, target, adjust, allCount, validCount); } return adjust; } - public float indexerScore(UTF8 url, Document doc, CrawlDatum dbDatum, CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) throws ScoringFilterException { + public float indexerScore(Text url, Document doc, CrawlDatum dbDatum, CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) throws ScoringFilterException { for (int i = 0; i < this.filters.length; i++) { initScore = this.filters[i].indexerScore(url, doc, dbDatum, fetchDatum, parse, inlinks, initScore); } Index: src/java/org/apache/nutch/crawl/CrawlDbReader.java =================================================================== --- src/java/org/apache/nutch/crawl/CrawlDbReader.java (revision 463985) +++ src/java/org/apache/nutch/crawl/CrawlDbReader.java (working copy) @@ -33,7 +33,7 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.JobClient; @@ -88,10 +88,10 @@ public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { CrawlDatum cd = (CrawlDatum) value; - output.collect(new UTF8("T"), COUNT_1); - output.collect(new UTF8("status " + cd.getStatus()), COUNT_1); - output.collect(new UTF8("retry " + cd.getRetriesSinceFetch()), COUNT_1); - output.collect(new UTF8("s"), new LongWritable((long) (cd.getScore() * 1000.0))); + output.collect(new Text("T"), COUNT_1); + output.collect(new Text("status " + cd.getStatus()), COUNT_1); + output.collect(new Text("retry " + cd.getRetriesSinceFetch()), COUNT_1); + output.collect(new Text("s"), new LongWritable((long) (cd.getScore() * 1000.0))); } } @@ -104,7 +104,7 @@ public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { val.set(0L); - String k = ((UTF8)key).toString(); + String k = ((Text)key).toString(); if (!k.equals("s")) { while (values.hasNext()) { LongWritable cnt = (LongWritable)values.next(); @@ -121,9 +121,9 @@ if (cnt.get() > max) max = cnt.get(); total += cnt.get(); } - output.collect(new UTF8("scn"), new LongWritable(min)); - output.collect(new UTF8("scx"), new LongWritable(max)); - output.collect(new UTF8("sct"), new LongWritable(total)); + output.collect(new Text("scn"), new LongWritable(min)); + output.collect(new Text("scx"), new LongWritable(max)); + output.collect(new Text("sct"), new LongWritable(total)); } } } @@ -134,7 +134,7 @@ public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { - String k = ((UTF8) key).toString(); + String k = ((Text) key).toString(); if (k.equals("T")) { // sum all values for this key long sum = 0; @@ -244,7 +244,7 @@ job.addInputPath(new Path(crawlDb, CrawlDatum.DB_DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(CrawlDatum.class); job.setMapperClass(CrawlDbStatMapper.class); @@ -253,7 +253,7 @@ job.setOutputPath(tmpFolder); job.setOutputFormat(SequenceFileOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); JobClient.runJob(job); @@ -262,7 +262,7 @@ FileSystem fileSystem = FileSystem.get(config); SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(config, tmpFolder); - UTF8 key = new UTF8(); + Text key = new Text(); LongWritable value = new LongWritable(); TreeMap stats = new TreeMap(); @@ -315,7 +315,7 @@ } public CrawlDatum get(String crawlDb, String url, Configuration config) throws IOException { - UTF8 key = new UTF8(url); + Text key = new Text(url); CrawlDatum val = new CrawlDatum(); openReaders(crawlDb, config); CrawlDatum res = (CrawlDatum)MapFileOutputFormat.getEntry(readers, new HashPartitioner(), key, val); @@ -346,12 +346,12 @@ job.addInputPath(new Path(crawlDb, CrawlDatum.DB_DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(CrawlDatum.class); job.setOutputPath(outFolder); job.setOutputFormat(TextOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(CrawlDatum.class); JobClient.runJob(job); @@ -375,7 +375,7 @@ job.setJobName("topN prepare " + crawlDb); job.addInputPath(new Path(crawlDb, CrawlDatum.DB_DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(CrawlDatum.class); job.setMapperClass(CrawlDbTopNMapper.class); job.setReducerClass(IdentityReducer.class); @@ -383,7 +383,7 @@ job.setOutputPath(tempDir); job.setOutputFormat(SequenceFileOutputFormat.class); job.setOutputKeyClass(FloatWritable.class); - job.setOutputValueClass(UTF8.class); + job.setOutputValueClass(Text.class); // XXX hmmm, no setFloat() in the API ... :( job.setLong("CrawlDbReader.topN.min", Math.round(1000000.0 * min)); @@ -399,14 +399,14 @@ job.addInputPath(tempDir); job.setInputFormat(SequenceFileInputFormat.class); job.setInputKeyClass(FloatWritable.class); - job.setInputValueClass(UTF8.class); + job.setInputValueClass(Text.class); job.setMapperClass(IdentityMapper.class); job.setReducerClass(CrawlDbTopNReducer.class); job.setOutputPath(outFolder); job.setOutputFormat(TextOutputFormat.class); job.setOutputKeyClass(FloatWritable.class); - job.setOutputValueClass(UTF8.class); + job.setOutputValueClass(Text.class); // XXX *sigh* this apparently doesn't work ... :-(( job.setNumReduceTasks(1); // create a single file. Index: src/java/org/apache/nutch/crawl/LinkDb.java =================================================================== --- src/java/org/apache/nutch/crawl/LinkDb.java (revision 463985) +++ src/java/org/apache/nutch/crawl/LinkDb.java (working copy) @@ -30,13 +30,13 @@ import org.apache.hadoop.conf.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.nutch.net.URLFilters; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.parse.*; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; /** Maintains an inverted link map, listing incoming links for each url. */ public class LinkDb extends ToolBase implements Mapper, Reducer { @@ -161,7 +161,7 @@ anchor = anchor.substring(0, maxAnchorLength); } inlinks.add(new Inlink(fromUrl, anchor)); // collect inverted link - output.collect(new UTF8(toUrl), inlinks); + output.collect(new Text(toUrl), inlinks); } } @@ -256,7 +256,7 @@ job.setJobName("linkdb " + linkDb); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(ParseData.class); job.setMapperClass(LinkDb.class); @@ -277,7 +277,7 @@ job.setOutputPath(newLinkDb); job.setOutputFormat(MapFileOutputFormat.class); job.setBoolean("mapred.output.compress", true); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(Inlinks.class); return job; @@ -292,7 +292,7 @@ job.setJobName("linkdb merge " + linkDb); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(Inlinks.class); job.setMapperClass(LinkDbFilter.class); @@ -303,7 +303,7 @@ job.setOutputPath(newLinkDb); job.setOutputFormat(MapFileOutputFormat.class); job.setBoolean("mapred.output.compress", true); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(Inlinks.class); return job; Index: src/java/org/apache/nutch/crawl/LinkDbMerger.java =================================================================== --- src/java/org/apache/nutch/crawl/LinkDbMerger.java (revision 463985) +++ src/java/org/apache/nutch/crawl/LinkDbMerger.java (working copy) @@ -17,6 +17,8 @@ import java.util.ArrayList; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -24,8 +26,8 @@ import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.ToolBase; /** * This tool merges several LinkDb-s into one, optionally filtering @@ -46,6 +48,8 @@ * @author Andrzej Bialecki */ public class LinkDbMerger extends ToolBase { + private static final Log LOG = LogFactory.getLog(LinkDbMerger.class); + public LinkDbMerger() { } Index: src/java/org/apache/nutch/crawl/MapWritable.java =================================================================== --- src/java/org/apache/nutch/crawl/MapWritable.java (revision 463985) +++ src/java/org/apache/nutch/crawl/MapWritable.java (working copy) @@ -39,7 +39,7 @@ import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.ObjectWritable; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.StringUtils; @@ -81,7 +81,7 @@ addToMap(NullWritable.class, new Byte((byte) -127)); addToMap(LongWritable.class, new Byte((byte) -126)); - addToMap(UTF8.class, new Byte((byte) -125)); + addToMap(Text.class, new Byte((byte) -125)); addToMap(MD5Hash.class, new Byte((byte) -124)); addToMap(org.apache.nutch.fetcher.FetcherOutput.class, new Byte((byte) -123)); @@ -305,7 +305,7 @@ ClassIdEntry entry = fIdFirst; while (entry != null) { out.writeByte(entry.fId); - UTF8.writeString(out, entry.fclazz.getName()); + Text.writeString(out, entry.fclazz.getName()); entry = entry.fNextIdEntry; } } @@ -336,7 +336,7 @@ for (int i = 0; i < fIdCount; i++) { try { id = in.readByte(); - clazz = Class.forName(UTF8.readString(in)); + clazz = Class.forName(Text.readString(in)); addIdEntry(id, clazz); } catch (Exception e) { if (LOG.isWarnEnabled()) { Index: src/java/org/apache/nutch/crawl/Injector.java =================================================================== --- src/java/org/apache/nutch/crawl/Injector.java (revision 463985) +++ src/java/org/apache/nutch/crawl/Injector.java (working copy) @@ -28,13 +28,13 @@ import org.apache.hadoop.conf.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.nutch.net.*; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; /** This class takes a flat file of URLs and adds them to the of pages to be * crawled. Useful for bootstrapping the system. */ @@ -65,7 +65,7 @@ public void map(WritableComparable key, Writable val, OutputCollector output, Reporter reporter) throws IOException { - UTF8 value = (UTF8)val; + Text value = (Text)val; String url = value.toString(); // value is line of text // System.out.println("url: " +url); try { @@ -138,7 +138,7 @@ sortJob.setOutputPath(tempDir); sortJob.setOutputFormat(SequenceFileOutputFormat.class); - sortJob.setOutputKeyClass(UTF8.class); + sortJob.setOutputKeyClass(Text.class); sortJob.setOutputValueClass(CrawlDatum.class); JobClient.runJob(sortJob); Index: src/java/org/apache/nutch/crawl/CrawlDb.java =================================================================== --- src/java/org/apache/nutch/crawl/CrawlDb.java (revision 463985) +++ src/java/org/apache/nutch/crawl/CrawlDb.java (working copy) @@ -28,10 +28,10 @@ import org.apache.hadoop.conf.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; /** * This class takes the output of the fetcher and updates the @@ -97,15 +97,13 @@ job.addInputPath(current); } job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); - job.setInputValueClass(CrawlDatum.class); job.setMapperClass(CrawlDbFilter.class); job.setReducerClass(CrawlDbReducer.class); job.setOutputPath(newCrawlDb); job.setOutputFormat(MapFileOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(CrawlDatum.class); return job; Index: src/java/org/apache/nutch/crawl/CrawlDbMerger.java =================================================================== --- src/java/org/apache/nutch/crawl/CrawlDbMerger.java (revision 463985) +++ src/java/org/apache/nutch/crawl/CrawlDbMerger.java (working copy) @@ -27,14 +27,14 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.nutch.net.URLFilters; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; /** * This tool merges several CrawlDb-s into one, optionally filtering @@ -119,7 +119,7 @@ job.setJobName("crawldb merge " + output); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(CrawlDatum.class); job.setMapperClass(CrawlDbFilter.class); @@ -129,7 +129,7 @@ job.setOutputPath(newCrawlDb); job.setOutputFormat(MapFileOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(CrawlDatum.class); return job; Index: src/java/org/apache/nutch/crawl/CrawlDbFilter.java =================================================================== --- src/java/org/apache/nutch/crawl/CrawlDbFilter.java (revision 463985) +++ src/java/org/apache/nutch/crawl/CrawlDbFilter.java (working copy) @@ -20,7 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.JobConf; @@ -71,6 +71,8 @@ } public void close() {} + + private Text newKey = new Text(); public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { @@ -92,7 +94,6 @@ } } if (url != null) { // if it passes - UTF8 newKey = (UTF8) key; newKey.set(url); // collect it output.collect(newKey, value); } Index: src/java/org/apache/nutch/crawl/Inlink.java =================================================================== --- src/java/org/apache/nutch/crawl/Inlink.java (revision 463985) +++ src/java/org/apache/nutch/crawl/Inlink.java (working copy) @@ -33,19 +33,19 @@ } public void readFields(DataInput in) throws IOException { - fromUrl = UTF8.readString(in); - anchor = UTF8.readString(in); + fromUrl = Text.readString(in); + anchor = Text.readString(in); } /** Skips over one Inlink in the input. */ public static void skip(DataInput in) throws IOException { - UTF8.skip(in); // skip fromUrl - UTF8.skip(in); // skip anchor + Text.skip(in); // skip fromUrl + Text.skip(in); // skip anchor } public void write(DataOutput out) throws IOException { - UTF8.writeString(out, fromUrl); - UTF8.writeString(out, anchor); + Text.writeString(out, fromUrl); + Text.writeString(out, anchor); } public static Inlink read(DataInput in) throws IOException { Index: src/java/org/apache/nutch/crawl/PartitionUrlByHost.java =================================================================== --- src/java/org/apache/nutch/crawl/PartitionUrlByHost.java (revision 463985) +++ src/java/org/apache/nutch/crawl/PartitionUrlByHost.java (working copy) @@ -42,7 +42,7 @@ /** Hash by hostname. */ public int getPartition(WritableComparable key, Writable value, int numReduceTasks) { - String urlString = ((UTF8)key).toString(); + String urlString = ((Text)key).toString(); try { urlString = normalizers.normalize(urlString, URLNormalizers.SCOPE_PARTITION); } catch (Exception e) { Index: src/java/org/apache/nutch/crawl/Generator.java =================================================================== --- src/java/org/apache/nutch/crawl/Generator.java (revision 463985) +++ src/java/org/apache/nutch/crawl/Generator.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.hadoop.fs.Path; import org.apache.nutch.net.URLFilterException; @@ -38,7 +39,6 @@ import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; /** Generates a subset of a crawl db to fetch. */ public class Generator extends ToolBase { @@ -46,11 +46,11 @@ public static final Log LOG = LogFactory.getLog(Generator.class); public static class SelectorEntry implements Writable { - public UTF8 url; + public Text url; public CrawlDatum datum; public SelectorEntry() { - url = new UTF8(); + url = new Text(); datum = new CrawlDatum(); } @@ -102,7 +102,7 @@ public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { - UTF8 url = (UTF8)key; + Text url = (Text)key; // don't generate URLs that don't pass URLFilters try { if (filters.filter(url.toString()) == null) @@ -122,7 +122,7 @@ float sort = 1.0f; try { - sort = scfilters.generatorSortValue((UTF8)key, crawlDatum, sort); + sort = scfilters.generatorSortValue((Text)key, crawlDatum, sort); } catch (ScoringFilterException sfe) { if (LOG.isWarnEnabled()) { LOG.warn("Couldn't filter generatorSortValue for " + key + ": " + sfe); @@ -131,7 +131,7 @@ // sort by decreasing score, using DecreasingFloatComparator sortValue.set(sort); entry.datum = crawlDatum; - entry.url = (UTF8)key; + entry.url = (Text)key; output.collect(sortValue, entry); // invert for sort by score } @@ -150,7 +150,7 @@ while (values.hasNext() && count < limit) { SelectorEntry entry = (SelectorEntry)values.next(); - UTF8 url = entry.url; + Text url = entry.url; if (maxPerHost > 0) { // are we counting hosts? String host = new URL(url.toString()).getHost(); @@ -236,11 +236,11 @@ /** Sort fetch lists by hash of URL. */ public static class HashComparator extends WritableComparator { - public HashComparator() { super(UTF8.class); } + public HashComparator() { super(Text.class); } public int compare(WritableComparable a, WritableComparable b) { - UTF8 url1 = (UTF8)a; - UTF8 url2 = (UTF8)b; + Text url1 = (Text)a; + Text url2 = (Text)b; int hash1 = hash(url1.getBytes(), 0, url1.getLength()); int hash2 = hash(url2.getBytes(), 0, url2.getLength()); if (hash1 != hash2) { @@ -252,14 +252,12 @@ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - int n1 = readUnsignedShort(b1, s1); - int n2 = readUnsignedShort(b2, s2); - int hash1 = hash(b1, s1+2, n1); - int hash2 = hash(b2, s2+2, n2); + int hash1 = hash(b1, s1, l1); + int hash2 = hash(b2, s2, l2); if (hash1 != hash2) { return hash1 - hash2; } - return compareBytes(b1, s1+2, n1, b2, s2+2, n2); + return compareBytes(b1, s1, l1, b2, s2, l2); } private static int hash(byte[] bytes, int start, int length) { @@ -319,7 +317,7 @@ job.setInputPath(new Path(dbDir, CrawlDatum.DB_DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(CrawlDatum.class); job.setMapperClass(Selector.class); @@ -353,7 +351,7 @@ job.setOutputPath(output); job.setOutputFormat(SequenceFileOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(CrawlDatum.class); job.setOutputKeyComparatorClass(HashComparator.class); JobClient.runJob(job); Index: src/java/org/apache/nutch/crawl/LinkDbReader.java =================================================================== --- src/java/org/apache/nutch/crawl/LinkDbReader.java (revision 463985) +++ src/java/org/apache/nutch/crawl/LinkDbReader.java (working copy) @@ -27,11 +27,11 @@ import org.apache.hadoop.mapred.*; import org.apache.hadoop.mapred.lib.HashPartitioner; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ToolBase; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.ToolBase; import java.util.Iterator; @@ -59,14 +59,14 @@ this.directory = directory; } - public String[] getAnchors(UTF8 url) throws IOException { + public String[] getAnchors(Text url) throws IOException { Inlinks inlinks = getInlinks(url); if (inlinks == null) return null; return inlinks.getAnchors(); } - public Inlinks getInlinks(UTF8 url) throws IOException { + public Inlinks getInlinks(Text url) throws IOException { if (readers == null) { synchronized(this) { @@ -100,12 +100,12 @@ job.addInputPath(new Path(linkdb, LinkDb.CURRENT_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(Inlinks.class); job.setOutputPath(outFolder); job.setOutputFormat(TextOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(Inlinks.class); JobClient.runJob(job); @@ -129,7 +129,7 @@ return 0; } else if (args[1].equals("-url")) { init(new Path(args[0])); - Inlinks links = getInlinks(new UTF8(args[2])); + Inlinks links = getInlinks(new Text(args[2])); if (links == null) { System.out.println(" - no link information."); } else { Index: src/java/org/apache/nutch/crawl/CrawlDbReducer.java =================================================================== --- src/java/org/apache/nutch/crawl/CrawlDbReducer.java (revision 463985) +++ src/java/org/apache/nutch/crawl/CrawlDbReducer.java (working copy) @@ -109,7 +109,7 @@ } else { result.setStatus(CrawlDatum.STATUS_DB_UNFETCHED); try { - scfilters.initialScore((UTF8)key, result); + scfilters.initialScore((Text)key, result); } catch (ScoringFilterException e) { if (LOG.isWarnEnabled()) { LOG.warn("Cannot filter init score for url " + key + @@ -152,7 +152,7 @@ } try { - scfilters.updateDbScore((UTF8)key, old, result, linked); + scfilters.updateDbScore((Text)key, old, result, linked); } catch (Exception e) { if (LOG.isWarnEnabled()) { LOG.warn("Couldn't update score, key=" + key + ": " + e); Index: src/java/org/apache/nutch/parse/ParseOutputFormat.java =================================================================== --- src/java/org/apache/nutch/parse/ParseOutputFormat.java (revision 463985) +++ src/java/org/apache/nutch/parse/ParseOutputFormat.java (working copy) @@ -66,13 +66,13 @@ new Path(new Path(job.getOutputPath(), CrawlDatum.PARSE_DIR_NAME), name); final MapFile.Writer textOut = - new MapFile.Writer(fs, text.toString(), UTF8.class, ParseText.class); + new MapFile.Writer(fs, text.toString(), Text.class, ParseText.class); final MapFile.Writer dataOut = - new MapFile.Writer(fs, data.toString(), UTF8.class,ParseData.class,true); + new MapFile.Writer(fs, data.toString(), Text.class,ParseData.class,true); final SequenceFile.Writer crawlOut = - new SequenceFile.Writer(fs, crawl, UTF8.class, CrawlDatum.class); + new SequenceFile.Writer(fs, crawl, Text.class, CrawlDatum.class); return new RecordWriter() { @@ -141,10 +141,10 @@ } } CrawlDatum target = new CrawlDatum(CrawlDatum.STATUS_LINKED, interval); - UTF8 targetUrl = new UTF8(toUrls[i]); + Text targetUrl = new Text(toUrls[i]); adjust = null; try { - adjust = scfilters.distributeScoreToOutlink((UTF8)key, targetUrl, + adjust = scfilters.distributeScoreToOutlink((Text)key, targetUrl, parseData, target, null, links.length, validCount); } catch (ScoringFilterException e) { if (LOG.isWarnEnabled()) { Index: src/java/org/apache/nutch/parse/ParseData.java =================================================================== --- src/java/org/apache/nutch/parse/ParseData.java (revision 463985) +++ src/java/org/apache/nutch/parse/ParseData.java (working copy) @@ -34,7 +34,7 @@ public final class ParseData extends VersionedWritable implements Configurable { public static final String DIR_NAME = "parse_data"; - private final static byte VERSION = 4; + private final static byte VERSION = 5; private String title; private Outlink[] outlinks; @@ -42,6 +42,7 @@ private Metadata parseMeta; private ParseStatus status; private Configuration conf; + private byte version = VERSION; // TODO mb@media-style.com: should we really implement Configurable or should we add the // parameter Configuration to the default-constructor. NOTE: The test @@ -110,16 +111,16 @@ // Writable methods // - public byte getVersion() { return VERSION; } + public byte getVersion() { return version; } public final void readFields(DataInput in) throws IOException { - byte version = in.readByte(); - if (version > 1) - status = ParseStatus.read(in); - else - status = ParseStatus.STATUS_SUCCESS; - title = UTF8.readString(in); // read title + version = in.readByte(); + // incompatible change from UTF8 (version < 5) to Text + if (version != VERSION) + throw new VersionMismatchException(VERSION, version); + status = ParseStatus.read(in); + title = Text.readString(in); // read title int totalOutlinks = in.readInt(); // read outlinks int maxOutlinksPerPage = this.conf.getInt("db.max.outlinks.per.page", 100); @@ -139,7 +140,7 @@ int propertyCount = in.readInt(); // read metadata contentMeta = new Metadata(); for (int i = 0; i < propertyCount; i++) { - contentMeta.add(UTF8.readString(in), UTF8.readString(in)); + contentMeta.add(Text.readString(in), Text.readString(in)); } } else { contentMeta = new Metadata(); @@ -154,7 +155,7 @@ public final void write(DataOutput out) throws IOException { out.writeByte(VERSION); // write version status.write(out); // write status - UTF8.writeString(out, title); // write title + Text.writeString(out, title); // write title out.writeInt(outlinks.length); // write outlinks for (int i = 0; i < outlinks.length; i++) { @@ -189,6 +190,7 @@ public String toString() { StringBuffer buffer = new StringBuffer(); + buffer.append("Version: " + version + "\n" ); buffer.append("Status: " + status + "\n" ); buffer.append("Title: " + title + "\n" ); Index: src/java/org/apache/nutch/parse/ParserChecker.java =================================================================== --- src/java/org/apache/nutch/parse/ParserChecker.java (revision 463985) +++ src/java/org/apache/nutch/parse/ParserChecker.java (working copy) @@ -24,7 +24,7 @@ import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.crawl.CrawlDatum; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.parse.ParseUtil; import org.apache.nutch.protocol.ProtocolFactory; @@ -75,7 +75,7 @@ Configuration conf = NutchConfiguration.create(); ProtocolFactory factory = new ProtocolFactory(conf); Protocol protocol = factory.getProtocol(url); - Content content = protocol.getProtocolOutput(new UTF8(url), new CrawlDatum()).getContent(); + Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent(); if (force) { content.setContentType(contentType); Index: src/java/org/apache/nutch/parse/Outlink.java =================================================================== --- src/java/org/apache/nutch/parse/Outlink.java (revision 463985) +++ src/java/org/apache/nutch/parse/Outlink.java (working copy) @@ -37,19 +37,19 @@ } public void readFields(DataInput in) throws IOException { - toUrl = UTF8.readString(in); - anchor = UTF8.readString(in); + toUrl = Text.readString(in); + anchor = Text.readString(in); } /** Skips over one Outlink in the input. */ public static void skip(DataInput in) throws IOException { - UTF8.skip(in); // skip toUrl - UTF8.skip(in); // skip anchor + Text.skip(in); // skip toUrl + Text.skip(in); // skip anchor } public void write(DataOutput out) throws IOException { - UTF8.writeString(out, toUrl); - UTF8.writeString(out, anchor); + Text.writeString(out, toUrl); + Text.writeString(out, anchor); } public static Outlink read(DataInput in) throws IOException { Index: src/java/org/apache/nutch/parse/ParseSegment.java =================================================================== --- src/java/org/apache/nutch/parse/ParseSegment.java (revision 463985) +++ src/java/org/apache/nutch/parse/ParseSegment.java (working copy) @@ -54,10 +54,17 @@ } public void close() {} + + private Text newKey = new Text(); public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { + // convert on the fly from old UTF8 keys + if (key instanceof UTF8) { + newKey.set(key.toString()); + key = newKey; + } Content content = (Content)value; Parse parse = null; @@ -75,7 +82,7 @@ if (status.isSuccess()) { try { - scfilters.passScoreAfterParsing((UTF8)key, content, parse); + scfilters.passScoreAfterParsing((Text)key, content, parse); } catch (ScoringFilterException e) { if (LOG.isWarnEnabled()) { e.printStackTrace(LogUtil.getWarnStream(LOG)); @@ -107,14 +114,14 @@ job.setInputPath(new Path(segment, Content.DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); - job.setInputKeyClass(UTF8.class); + job.setInputKeyClass(Text.class); job.setInputValueClass(Content.class); job.setMapperClass(ParseSegment.class); job.setReducerClass(ParseSegment.class); job.setOutputPath(segment); job.setOutputFormat(ParseOutputFormat.class); - job.setOutputKeyClass(UTF8.class); + job.setOutputKeyClass(Text.class); job.setOutputValueClass(ParseImpl.class); JobClient.runJob(job); Index: src/java/org/apache/nutch/util/ToolBase.java =================================================================== --- src/java/org/apache/nutch/util/ToolBase.java (revision 463985) +++ src/java/org/apache/nutch/util/ToolBase.java (working copy) @@ -1,148 +0,0 @@ -/** - * Copyright 2005 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nutch.util; - -import java.io.IOException; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.Tool; - -/** - * This is a verbatim copy of org.apache.hadoop.util.ToolBase, with - * a temporary workaround for Hadoop's ToolBase.doMain() deficiency. - * This class should be removed when HADOOP-488 is fixed, and all its - * uses should be replaced with org.apache.hadoop.util.ToolBase. - * - * @author Andrzej Bialecki - */ -public abstract class ToolBase implements Tool { - public static final Log LOG = LogFactory.getLog( - "org.apache.nutch.util.ToolBase"); - public Configuration conf; - - public void setConf(Configuration conf) { - this.conf = conf; - } - - public Configuration getConf() { - return conf; - } - - /* - * Specify properties of each generic option - */ - static private Options buildGeneralOptions() { - Option fs = OptionBuilder.withArgName("local|namenode:port") - .hasArg() - .withDescription("specify a namenode") - .create("fs"); - Option jt = OptionBuilder.withArgName("local|jobtracker:port") - .hasArg() - .withDescription("specify a job tracker") - .create("jt"); - Option oconf = OptionBuilder.withArgName("configuration file") - .hasArg() - .withDescription("specify an application configuration file" ) - .create("conf"); - Option property = OptionBuilder.withArgName("property=value") - .hasArgs() - .withArgPattern("=", 1) - .withDescription("use value for given property") - .create('D'); - Options opts = new Options(); - opts.addOption(fs); - opts.addOption(jt); - opts.addOption(oconf); - opts.addOption(property); - - return opts; - } - - /* - * Modify configuration according user-specified generic options - * @param conf Configuration to be modified - * @param line User-specified generic options - */ - static private void processGeneralOptions( Configuration conf, - CommandLine line ) { - if(line.hasOption("fs")) { - conf.set("fs.default.name", line.getOptionValue("fs")); - } - - if(line.hasOption("jt")) { - conf.set("mapred.job.tracker", line.getOptionValue("jt")); - } - if(line.hasOption("conf")) { - conf.addFinalResource(new Path(line.getOptionValue("conf"))); - } - if(line.hasOption('D')) { - String[] property = line.getOptionValues('D'); - for(int i=0; i - + Index: src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java =================================================================== --- src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java (revision 463985) +++ src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java (working copy) @@ -20,7 +20,7 @@ import java.io.InputStreamReader; import org.apache.nutch.crawl.CrawlDatum; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.protocol.ProtocolFactory; import org.apache.nutch.protocol.Protocol; import org.apache.nutch.protocol.Content; @@ -85,7 +85,7 @@ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); parse = new ParseUtil(conf).parse(content); Index: src/plugin/parse-msexcel/src/test/org/apache/nutch/parse/msexcel/TestMSExcelParser.java =================================================================== --- src/plugin/parse-msexcel/src/test/org/apache/nutch/parse/msexcel/TestMSExcelParser.java (revision 463985) +++ src/plugin/parse-msexcel/src/test/org/apache/nutch/parse/msexcel/TestMSExcelParser.java (working copy) @@ -20,7 +20,7 @@ // Hadoop imports import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; /** @@ -61,7 +61,7 @@ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; protocol = factory.getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); parse = parser.parseByExtensionId("parse-msexcel", content); Index: src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java =================================================================== --- src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java (revision 463985) +++ src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java (working copy) @@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import junit.framework.TestCase; @@ -68,7 +68,7 @@ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-zip",content); assertTrue(parse.getText().equals(expectedText)); } Index: src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java (revision 463985) +++ src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java (working copy) @@ -158,6 +158,7 @@ } if (curRules == EMPTY_RULES || curRules == null) { LOG.warn("can't find rules for scope '" + scope + "', using default"); + scopedRules.put(scope, EMPTY_RULES); } } if (curRules == EMPTY_RULES || curRules == null) { Index: src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java =================================================================== --- src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java (revision 463985) +++ src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java (working copy) @@ -20,7 +20,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.nutch.crawl.CrawlDatum; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.net.protocols.HttpDateFormat; import org.apache.nutch.net.protocols.Response; @@ -67,7 +67,7 @@ /** Set the point at which content is truncated. */ public void setMaxContentLength(int length) {maxContentLength = length;} - public ProtocolOutput getProtocolOutput(UTF8 url, CrawlDatum datum) { + public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) { String urlString = url.toString(); try { URL u = new URL(urlString); @@ -141,7 +141,7 @@ // set log level //LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase())); - Content content = file.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + Content content = file.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); System.err.println("Content-Type: " + content.getContentType()); System.err.println("Content-Length: " + Index: src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java =================================================================== --- src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java (revision 463985) +++ src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java (working copy) @@ -29,7 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import junit.framework.TestCase; @@ -86,7 +86,7 @@ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-rss", content); //check that there are 3 outlinks: Index: src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java =================================================================== --- src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java (revision 463985) +++ src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java (working copy) @@ -27,7 +27,7 @@ import org.apache.commons.logging.LogFactory; // Hadoop imports -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.hadoop.conf.Configuration; // Nutch imports @@ -216,7 +216,7 @@ RSSParser parser = new RSSParser(); parser.setConf(conf); Protocol protocol = new ProtocolFactory(conf).getProtocol(url); - Content content = protocol.getProtocolOutput(new UTF8(url), new CrawlDatum()).getContent(); + Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent(); Parse parse = parser.getParse(content); System.out.println("data: "+ parse.getData()); System.out.println("text: "+parse.getText()); Index: src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java =================================================================== --- src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java (revision 463985) +++ src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java (working copy) @@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import junit.framework.TestCase; @@ -68,7 +68,7 @@ Configuration conf = NutchConfiguration.create(); protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-pdf", content); int index = parse.getText().indexOf(expectedText); Index: src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java =================================================================== --- src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (revision 463985) +++ src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (working copy) @@ -40,7 +40,7 @@ // Hadoop imports import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; /** @@ -170,7 +170,7 @@ - public ProtocolOutput getProtocolOutput(UTF8 url, CrawlDatum datum) { + public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) { String urlString = url.toString(); try { @@ -509,7 +509,7 @@ // LOGGER.setLevel(Level.FINE); // } - ProtocolOutput out = http.getProtocolOutput(new UTF8(url), new CrawlDatum()); + ProtocolOutput out = http.getProtocolOutput(new Text(url), new CrawlDatum()); Content content = out.getContent(); System.out.println("Status: " + out.getStatus()); Index: src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-1.9-rc1-dev.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.0.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.0.0.jar ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Index: src/plugin/lib-lucene-analyzers/plugin.xml =================================================================== --- src/plugin/lib-lucene-analyzers/plugin.xml (revision 463985) +++ src/plugin/lib-lucene-analyzers/plugin.xml (working copy) @@ -10,11 +10,11 @@ - + Index: src/plugin/parse-oo/src/test/org/apache/nutch/parse/oo/TestOOParser.java =================================================================== --- src/plugin/parse-oo/src/test/org/apache/nutch/parse/oo/TestOOParser.java (revision 463985) +++ src/plugin/parse-oo/src/test/org/apache/nutch/parse/oo/TestOOParser.java (working copy) @@ -21,7 +21,7 @@ import org.apache.nutch.crawl.CrawlDatum; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.protocol.*; import org.apache.nutch.parse.Parse; @@ -87,7 +87,7 @@ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; protocol = factory.getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); parse = parser.getParse(content); Index: src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java =================================================================== --- src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (revision 463985) +++ src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (working copy) @@ -22,7 +22,7 @@ import org.apache.commons.net.ftp.FTPFileEntryParser; import org.apache.nutch.crawl.CrawlDatum; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.net.protocols.HttpDateFormat; import org.apache.nutch.net.protocols.Response; @@ -111,7 +111,7 @@ this.keepConnection = keepConnection; } - public ProtocolOutput getProtocolOutput(UTF8 url, CrawlDatum datum) { + public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) { String urlString = url.toString(); try { URL u = new URL(urlString); @@ -207,7 +207,7 @@ // set log level //LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase())); - Content content = ftp.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + Content content = ftp.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); System.err.println("Content-Type: " + content.getContentType()); System.err.println("Content-Length: " + Index: src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java =================================================================== --- src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java (revision 463985) +++ src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java (working copy) @@ -24,7 +24,7 @@ import org.apache.nutch.indexer.IndexingFilter; import org.apache.nutch.indexer.IndexingException; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.Inlinks; @@ -49,7 +49,7 @@ private Configuration conf; - public Document filter(Document doc, Parse parse, UTF8 url, CrawlDatum datum, Inlinks inlinks) + public Document filter(Document doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Metadata metadata = parse.getData().getParseMeta(); Index: src/plugin/parse-rtf/src/test/org/apache/nutch/parse/rtf/TestRTFParser.java =================================================================== --- src/plugin/parse-rtf/src/test/org/apache/nutch/parse/rtf/TestRTFParser.java (revision 463985) +++ src/plugin/parse-rtf/src/test/org/apache/nutch/parse/rtf/TestRTFParser.java (working copy) @@ -34,7 +34,7 @@ // Hadoop imports import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; /** @@ -72,7 +72,7 @@ Configuration conf = NutchConfiguration.create(); urlString = "file:" + sampleDir + fileSeparator + rtfFile; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()) + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()) .getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-rtf", content); String text = parse.getText(); Index: src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java =================================================================== --- src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java (revision 463985) +++ src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java (working copy) @@ -17,7 +17,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -66,7 +66,7 @@ doc.add(new Field(FIELD_NAME, collname, Field.Store.YES, Field.Index.TOKENIZED)); } - public Document filter(Document doc, Parse parse, UTF8 url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { + public Document filter(Document doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { String sUrl = url.toString(); addSubCollectionField(doc, sUrl); return doc; Index: src/plugin/parse-mp3/src/test/org/apache/nutch/parse/mp3/TestMP3Parser.java =================================================================== --- src/plugin/parse-mp3/src/test/org/apache/nutch/parse/mp3/TestMP3Parser.java (revision 463985) +++ src/plugin/parse-mp3/src/test/org/apache/nutch/parse/mp3/TestMP3Parser.java (working copy) @@ -17,7 +17,7 @@ package org.apache.nutch.parse.mp3; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.parse.Parse; @@ -68,7 +68,7 @@ Configuration conf = NutchConfiguration.create(); urlString = "file:" + sampleDir + fileSeparator + id3v2; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()) + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()) .getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-mp3", content); Metadata metadata = parse.getData().getParseMeta(); @@ -100,7 +100,7 @@ Configuration conf = NutchConfiguration.create(); urlString = "file:" + sampleDir + fileSeparator + id3v1; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()) + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()) .getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-mp3", content); @@ -127,7 +127,7 @@ Configuration conf = NutchConfiguration.create(); urlString = "file:" + sampleDir + fileSeparator + none; protocol = new ProtocolFactory(conf).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()) + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()) .getContent(); parse = new ParseUtil(conf).parseByExtensionId("parse-mp3", content); // Metadata metadata = parse.getData().getParseMeta(); Index: src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java =================================================================== --- src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java (revision 463985) +++ src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java (working copy) @@ -48,7 +48,7 @@ import org.apache.nutch.util.mime.MimeTypeException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -81,7 +81,7 @@ /** Get the MimeTypes resolver instance. */ private MimeTypes MIME; - public Document filter(Document doc, Parse parse, UTF8 url, CrawlDatum datum, Inlinks inlinks) + public Document filter(Document doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { String url_s = url.toString(); Index: src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java =================================================================== --- src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java (revision 463985) +++ src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java (working copy) @@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import junit.framework.TestCase; @@ -82,7 +82,7 @@ // get nutch content Protocol protocol = new ProtocolFactory(NutchConfiguration.create()).getProtocol(urlString); - content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent(); + content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); protocol = null; } Index: src/plugin/parse-mspowerpoint/src/test/org/apache/nutch/parse/mspowerpoint/TestMSPowerPointParser.java =================================================================== --- src/plugin/parse-mspowerpoint/src/test/org/apache/nutch/parse/mspowerpoint/TestMSPowerPointParser.java (revision 463985) +++ src/plugin/parse-mspowerpoint/src/test/org/apache/nutch/parse/mspowerpoint/TestMSPowerPointParser.java (working copy) @@ -37,7 +37,7 @@ import org.apache.nutch.protocol.ProtocolFactory; import org.apache.nutch.util.NutchConfiguration; -import org.apache.hadoop.io.UTF8; +import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; /** @@ -107,7 +107,7 @@ System.out.println("Testing file: " + this.urlString + "..."); this.protocol =new ProtocolFactory(NutchConfiguration.create()).getProtocol(this.urlString); - this.content = this.protocol.getProtocolOutput(new UTF8(this.urlString), new CrawlDatum()).getContent(); + this.content = this.protocol.getProtocolOutput(new Text(this.urlString), new CrawlDatum()).getContent(); } /**