diff --git ivy/ivy.xml ivy/ivy.xml index 7d45e0f..eeabddb 100644 --- ivy/ivy.xml +++ ivy/ivy.xml @@ -102,8 +102,7 @@ - - + - - + diff --git src/gora/host.avsc src/gora/host.avsc index e4165cb..f7e386d 100644 --- src/gora/host.avsc +++ src/gora/host.avsc @@ -2,8 +2,8 @@ "type": "record", "namespace": "org.apache.nutch.storage", "fields": [ - {"name": "metadata", "type": {"type": "map", "values": "bytes"}}, - {"name": "outlinks", "type": {"type": "map", "values": "string"}}, - {"name": "inlinks", "type": {"type": "map", "values": "string"}} + {"name": "metadata", "type": {"type": "map", "values": "bytes"}, "default":{}}, + {"name": "outlinks", "type": {"type": "map", "values": "string"}, "default":{}}, + {"name": "inlinks", "type": {"type": "map", "values": "string"}, "default":{}} ] } diff --git src/gora/webpage.avsc src/gora/webpage.avsc index 601fae8..f025c74 100644 --- src/gora/webpage.avsc +++ src/gora/webpage.avsc @@ -2,47 +2,47 @@ "type": "record", "namespace": "org.apache.nutch.storage", "fields": [ - {"name": "baseUrl", "type": "string"}, - {"name": "status", "type": "int"}, - {"name": "fetchTime", "type": "long"}, - {"name": "prevFetchTime", "type": "long"}, - {"name": "fetchInterval", "type": "int"}, - {"name": "retriesSinceFetch", "type": "int"}, - {"name": "modifiedTime", "type": "long"}, - {"name": "prevModifiedTime", "type": "long"}, - {"name": "protocolStatus", "type": { + {"name": "baseUrl", "type": ["null","string"], "default":null}, + {"name": "status", "type": "int", "default":0}, + {"name": "fetchTime", "type": "long", "default":0}, + {"name": "prevFetchTime", "type": "long", "default":0}, + {"name": "fetchInterval", "type": "int", "default":0}, + {"name": "retriesSinceFetch", "type": "int", "default":0}, + {"name": "modifiedTime", "type": "long", "default":0}, + {"name": "prevModifiedTime", "type": "long", "default":0}, + {"name": "protocolStatus", "type": ["null", { "name": "ProtocolStatus", "type": "record", "namespace": "org.apache.nutch.storage", "fields": [ - {"name": "code", "type": "int"}, - {"name": "args", "type": {"type": "array", "items": "string"}}, - {"name": "lastModified", "type": "long"} + {"name": "code", "type": "int", "default":0}, + {"name": "args", "type": {"type": "array", "items": "string"}, "default":[]}, + {"name": "lastModified", "type": "long", "default":0} ] - }}, - {"name": "content", "type": "bytes"}, - {"name": "contentType", "type": "string"}, - {"name": "prevSignature", "type": "bytes"}, - {"name": "signature", "type": "bytes"}, - {"name": "title", "type": "string"}, - {"name": "text", "type": "string"}, - {"name": "parseStatus", "type": { + }], "default":null}, + {"name": "content", "type": ["null","bytes"], "default":null}, + {"name": "contentType", "type": ["null","string"], "default":null}, + {"name": "prevSignature", "type": ["null","bytes"], "default":null}, + {"name": "signature", "type": ["null","bytes"], "default":null}, + {"name": "title", "type": ["null","string"], "default":null}, + {"name": "text", "type": ["null","string"], "default":null}, + {"name": "parseStatus", "type": ["null", { "name": "ParseStatus", "type": "record", "namespace": "org.apache.nutch.storage", "fields": [ - {"name": "majorCode", "type": "int"}, - {"name": "minorCode", "type": "int"}, - {"name": "args", "type": {"type": "array", "items": "string"}} + {"name": "majorCode", "type": "int", "default":0}, + {"name": "minorCode", "type": "int", "default":0}, + {"name": "args", "type": {"type": "array", "items": "string"}, "default":[]} ] - }}, - {"name": "score", "type": "float"}, - {"name": "reprUrl", "type": "string"}, - {"name": "headers", "type": {"type": "map", "values": "string"}}, - {"name": "outlinks", "type": {"type": "map", "values": "string"}}, - {"name": "inlinks", "type": {"type": "map", "values": "string"}}, - {"name": "markers", "type": {"type": "map", "values": "string"}}, - {"name": "metadata", "type": {"type": "map", "values": "bytes"}}, - {"name": "batchId", "type": "string"} + }], "default":null}, + {"name": "score", "type": "float", "default":0}, + {"name": "reprUrl", "type": ["null","string"], "default":null}, + {"name": "headers", "type": {"type":"map", "values": ["null","string"]}, "default":{}}, + {"name": "outlinks", "type": {"type": "map", "values": ["null","string"]}, "default":{}}, + {"name": "inlinks", "type": {"type": "map", "values": ["null","string"]}, "default":{}}, + {"name": "markers", "type": {"type": "map", "values": ["null","string"]}, "default":{}}, + {"name": "metadata", "type": {"type": "map", "values": ["null","bytes"]}, "default":{}}, + {"name": "batchId", "type": ["null","string"], "default":null} ] } diff --git src/java/org/apache/nutch/api/DbReader.java src/java/org/apache/nutch/api/DbReader.java index 728da3a..68cc9ac 100644 --- src/java/org/apache/nutch/api/DbReader.java +++ src/java/org/apache/nutch/api/DbReader.java @@ -16,16 +16,7 @@ ******************************************************************************/ package org.apache.nutch.api; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeSet; - +import org.apache.avro.Schema; import org.apache.avro.util.Utf8; import org.apache.gora.query.Query; import org.apache.gora.query.Result; @@ -34,11 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.parse.ParseStatusUtils; import org.apache.nutch.protocol.ProtocolStatusUtils; -import org.apache.nutch.storage.Mark; -import org.apache.nutch.storage.ParseStatus; -import org.apache.nutch.storage.ProtocolStatus; -import org.apache.nutch.storage.StorageUtils; -import org.apache.nutch.storage.WebPage; +import org.apache.nutch.storage.*; import org.apache.nutch.util.Bytes; import org.apache.nutch.util.NutchJob; import org.apache.nutch.util.StringUtil; @@ -46,6 +33,11 @@ import org.apache.nutch.util.TableUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.Map.Entry; + public class DbReader { private static final Logger LOG = LoggerFactory.getLogger(DbReader.class); @@ -145,7 +137,7 @@ public class DbReader { public Map next() { url = res.getKey(); - page = (WebPage)res.get().clone(); + page = WebPage.newBuilder(res.get()).build(); try { advance(); if (!hasNext) { @@ -169,16 +161,16 @@ public class DbReader { if (fields == null || fields.contains("url")) { res.put("url", TableUtil.unreverseUrl(url)); } - String[] pfields = page.getFields(); - TreeSet flds = null; + List pfields = page.getSchema().getFields(); + TreeSet flds = null; if (fields != null) { - flds = (TreeSet)fields.clone(); + flds = (TreeSet) fields.clone(); } else { - flds = new TreeSet(Arrays.asList(pfields)); + flds = new TreeSet(pfields); } flds.retainAll(Arrays.asList(pfields)); - for (String f : flds) { - int idx = page.getFieldIndex(f); + for (Schema.Field f : flds) { + int idx = f.pos(); if (idx < 0) { continue; } @@ -187,43 +179,43 @@ public class DbReader { continue; } if ("metadata".equals(f)) { - Map metadata = page.getMetadata(); + Map metadata = page.getMetadata(); Map simpleMeta = new HashMap(); if (metadata != null) { - Iterator> iterator = metadata.entrySet() + Iterator> iterator = metadata.entrySet() .iterator(); while (iterator.hasNext()) { - Entry entry = iterator.next(); + Entry entry = iterator.next(); simpleMeta.put(entry.getKey().toString(), Bytes.toStringBinary(entry.getValue())); } } - res.put(f, simpleMeta); + res.put(f.name(), simpleMeta); } else if ("protocolStatus".equals(f)) { ProtocolStatus ps = page.getProtocolStatus(); - res.put(f, ProtocolStatusUtils.toString(ps)); + res.put(f.name(), ProtocolStatusUtils.toString(ps)); } else if ("parseStatus".equals(f)) { ParseStatus ps = page.getParseStatus(); - res.put(f, ParseStatusUtils.toString(ps)); + res.put(f.name(), ParseStatusUtils.toString(ps)); } else if ("signature".equals(f)) { ByteBuffer bb = page.getSignature(); - res.put(f, StringUtil.toHexString(bb)); + res.put(f.name(), StringUtil.toHexString(bb)); } else if ("content".equals(f)) { ByteBuffer bb = page.getContent(); - res.put(f, Bytes.toStringBinary(bb)); + res.put(f.name(), Bytes.toStringBinary(bb)); } else if ("markers".equals(f)) { - res.put(f, convertMap(page.getMarkers())); + res.put(f.name(), convertMap(page.getMarkers())); } else if ("inlinks".equals(f)) { - res.put(f, convertMap(page.getInlinks())); + res.put(f.name(), convertMap(page.getInlinks())); } else if ("outlinks".equals(f)) { - res.put(f, convertMap(page.getOutlinks())); + res.put(f.name(), convertMap(page.getOutlinks())); } else { if (val instanceof Utf8) { val = val.toString(); } else if (val instanceof ByteBuffer) { val = Bytes.toStringBinary((ByteBuffer)val); } - res.put(f, val); + res.put(f.name(), val); } } return res; diff --git src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java index 20ce90d..d9a9e8d 100755 --- src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java +++ src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java @@ -17,14 +17,14 @@ package org.apache.nutch.crawl; -import java.util.HashSet; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.nutch.storage.WebPage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Set; /** * This class provides common methods for implementations of @@ -196,7 +196,7 @@ implements FetchSchedule { // reduce fetchInterval so that it fits within the max value if (page.getFetchInterval() > maxInterval) page.setFetchInterval(Math.round(maxInterval * 0.9f)); - page.setStatus(CrawlStatus.STATUS_UNFETCHED); + page.setStatus((int) CrawlStatus.STATUS_UNFETCHED); page.setRetriesSinceFetch(0); // TODO: row.setSignature(null) ?? page.setModifiedTime(0L); diff --git src/java/org/apache/nutch/crawl/DbUpdateMapper.java src/java/org/apache/nutch/crawl/DbUpdateMapper.java index 5bef21d..48e4913 100644 --- src/java/org/apache/nutch/crawl/DbUpdateMapper.java +++ src/java/org/apache/nutch/crawl/DbUpdateMapper.java @@ -67,11 +67,11 @@ extends GoraMapper { String url = TableUtil.unreverseUrl(key); scoreData.clear(); - Map outlinks = page.getOutlinks(); + Map outlinks = page.getOutlinks(); if (outlinks != null) { - for (Entry e : outlinks.entrySet()) { + for (Entry e : outlinks.entrySet()) { int depth=Integer.MAX_VALUE; - Utf8 depthUtf8=page.getFromMarkers(DbUpdaterJob.DISTANCE); + CharSequence depthUtf8 = page.getMarkers().get(DbUpdaterJob.DISTANCE); if (depthUtf8 != null) depth=Integer.parseInt(depthUtf8.toString()); scoreData.add(new ScoreDatum(0.0f, e.getKey().toString(), e.getValue().toString(), depth)); diff --git src/java/org/apache/nutch/crawl/DbUpdateReducer.java src/java/org/apache/nutch/crawl/DbUpdateReducer.java index a671694..89e57f5 100644 --- src/java/org/apache/nutch/crawl/DbUpdateReducer.java +++ src/java/org/apache/nutch/crawl/DbUpdateReducer.java @@ -22,7 +22,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.avro.util.Utf8; -import org.slf4j.Logger; +import org.apache.gora.mapreduce.GoraReducer; +import org.apache.gora.store.DataStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.StringUtils; @@ -35,7 +36,7 @@ import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.TableUtil; import org.apache.nutch.util.WebPageWritable; -import org.apache.gora.mapreduce.GoraReducer; +import org.slf4j.Logger; public class DbUpdateReducer extends GoraReducer { @@ -97,16 +98,16 @@ extends GoraReducer { if (!additionsAllowed) { return; } - page = new WebPage(); + page = WebPage.newBuilder().build(); schedule.initializeSchedule(url, page); - page.setStatus(CrawlStatus.STATUS_UNFETCHED); + page.setStatus((int) CrawlStatus.STATUS_UNFETCHED); try { scoringFilters.initialScore(url, page); } catch (ScoringFilterException e) { page.setScore(0.0f); } } else { - byte status = (byte)page.getStatus(); + byte status = page.getStatus().byteValue(); switch (status) { case CrawlStatus.STATUS_FETCHED: // succesful fetch case CrawlStatus.STATUS_REDIR_TEMP: // successful fetch, redirected @@ -129,7 +130,7 @@ extends GoraReducer { long prevFetchTime = page.getPrevFetchTime(); long modifiedTime = page.getModifiedTime(); long prevModifiedTime = page.getPrevModifiedTime(); - Utf8 lastModified = page.getFromHeaders(new Utf8("Last-Modified")); + CharSequence lastModified = page.getHeaders().get(new Utf8("Last-Modified")); if ( lastModified != null ){ try { modifiedTime = HttpDateFormat.toLong(lastModified.toString()); @@ -145,9 +146,9 @@ extends GoraReducer { case CrawlStatus.STATUS_RETRY: schedule.setPageRetrySchedule(url, page, 0L, page.getPrevModifiedTime(), page.getFetchTime()); if (page.getRetriesSinceFetch() < retryMax) { - page.setStatus(CrawlStatus.STATUS_UNFETCHED); + page.setStatus((int)CrawlStatus.STATUS_UNFETCHED); } else { - page.setStatus(CrawlStatus.STATUS_GONE); + page.setStatus((int)CrawlStatus.STATUS_GONE); } break; case CrawlStatus.STATUS_GONE: @@ -171,15 +172,15 @@ extends GoraReducer { if (inlinkDist < smallestDist) { smallestDist=inlinkDist; } - page.putToInlinks(new Utf8(inlink.getUrl()), new Utf8(inlink.getAnchor())); + page.getInlinks().put(new Utf8(inlink.getUrl()), new Utf8(inlink.getAnchor())); } if (smallestDist != Integer.MAX_VALUE) { int oldDistance=Integer.MAX_VALUE; - Utf8 oldDistUtf8 = page.getFromMarkers(DbUpdaterJob.DISTANCE); + CharSequence oldDistUtf8 = page.getMarkers().get(DbUpdaterJob.DISTANCE); if (oldDistUtf8 != null)oldDistance=Integer.parseInt(oldDistUtf8.toString()); int newDistance = smallestDist+1; if (newDistance < oldDistance) { - page.putToMarkers(DbUpdaterJob.DISTANCE, new Utf8(Integer.toString(newDistance))); + page.getMarkers().put(DbUpdaterJob.DISTANCE, new Utf8(Integer.toString(newDistance))); } } @@ -193,8 +194,8 @@ extends GoraReducer { // clear markers // But only delete when they exist. This is much faster for the underlying // store. The markers are on the input anyway. - if (page.getFromMetadata(FetcherJob.REDIRECT_DISCOVERED) != null) { - page.removeFromMetadata(FetcherJob.REDIRECT_DISCOVERED); + if (page.getMetadata().get(FetcherJob.REDIRECT_DISCOVERED) != null) { + page.getMetadata().put(FetcherJob.REDIRECT_DISCOVERED, null); } Mark.GENERATE_MARK.removeMarkIfExist(page); Mark.FETCH_MARK.removeMarkIfExist(page); diff --git src/java/org/apache/nutch/crawl/GeneratorMapper.java src/java/org/apache/nutch/crawl/GeneratorMapper.java index 144784a..6af4e75 100644 --- src/java/org/apache/nutch/crawl/GeneratorMapper.java +++ src/java/org/apache/nutch/crawl/GeneratorMapper.java @@ -16,9 +16,8 @@ ******************************************************************************/ package org.apache.nutch.crawl; -import java.io.IOException; -import java.net.MalformedURLException; - +import org.apache.avro.util.Utf8; +import org.apache.gora.mapreduce.GoraMapper; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.crawl.GeneratorJob.SelectorEntry; import org.apache.nutch.net.URLFilterException; @@ -29,8 +28,11 @@ import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.TableUtil; -import org.apache.avro.util.Utf8; -import org.apache.gora.mapreduce.GoraMapper; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.util.HashMap; public class GeneratorMapper extends GoraMapper { @@ -59,7 +61,7 @@ extends GoraMapper { //filter on distance if (maxDistance > -1) { - Utf8 distanceUtf8 = page.getFromMarkers(DbUpdaterJob.DISTANCE); + CharSequence distanceUtf8 = page.getMarkers().get(DbUpdaterJob.DISTANCE); if (distanceUtf8 != null) { int distance=Integer.parseInt(distanceUtf8.toString()); if (distance > maxDistance) { diff --git src/java/org/apache/nutch/crawl/GeneratorReducer.java src/java/org/apache/nutch/crawl/GeneratorReducer.java index fd5bc31..92019dd 100644 --- src/java/org/apache/nutch/crawl/GeneratorReducer.java +++ src/java/org/apache/nutch/crawl/GeneratorReducer.java @@ -22,6 +22,10 @@ import java.util.HashMap; import java.util.Map; import org.apache.avro.util.Utf8; +import org.apache.gora.mapreduce.GoraReducer; +import org.apache.gora.query.Query; +import org.apache.gora.query.Result; +import org.apache.gora.store.DataStore; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.crawl.GeneratorJob.SelectorEntry; import org.apache.nutch.fetcher.FetcherJob.FetcherMapper; @@ -29,7 +33,6 @@ import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.TableUtil; import org.apache.nutch.util.URLUtil; -import org.apache.gora.mapreduce.GoraReducer; /** Reduce class for generate * diff --git src/java/org/apache/nutch/crawl/InjectorJob.java src/java/org/apache/nutch/crawl/InjectorJob.java index bfb5f6d..b852dd5 100644 --- src/java/org/apache/nutch/crawl/InjectorJob.java +++ src/java/org/apache/nutch/crawl/InjectorJob.java @@ -16,15 +16,6 @@ ******************************************************************************/ package org.apache.nutch.crawl; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.text.SimpleDateFormat; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - import org.apache.avro.util.Utf8; import org.apache.gora.mapreduce.GoraOutputFormat; import org.apache.gora.persistency.Persistent; @@ -47,15 +38,15 @@ import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.StorageUtils; import org.apache.nutch.storage.WebPage; -import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.NutchTool; -import org.apache.nutch.util.TableUtil; -import org.apache.nutch.util.TimingUtil; -import org.apache.nutch.util.ToolUtil; +import org.apache.nutch.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.*; + /** This class takes a flat file of URLs and adds them to the of pages to be * crawled. Useful for bootstrapping the system. * The URL files contain one URL per line, optionally followed by custom metadata @@ -160,7 +151,7 @@ public class InjectorJob extends NutchTool implements Tool { return; } else { // if it passes String reversedUrl = TableUtil.reverseUrl(url); // collect it - WebPage row = new WebPage(); + WebPage row = WebPage.newBuilder().build(); row.setFetchTime(curTime); row.setFetchInterval(customInterval); @@ -169,7 +160,7 @@ public class InjectorJob extends NutchTool implements Tool { while (keysIter.hasNext()) { String keymd = keysIter.next(); String valuemd = metadata.get(keymd); - row.putToMetadata(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes())); + row.getMetadata().put(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes())); } if (customScore != -1) @@ -186,7 +177,7 @@ public class InjectorJob extends NutchTool implements Tool { } } context.getCounter("injector", "urls_injected").increment(1); - row.putToMarkers(DbUpdaterJob.DISTANCE, new Utf8(String.valueOf(0))); + row.getMarkers().put(DbUpdaterJob.DISTANCE, new Utf8(String.valueOf(0))); Mark.INJECT_MARK.putMark(row, YES_STRING); context.write(reversedUrl, row); } diff --git src/java/org/apache/nutch/crawl/MD5Signature.java src/java/org/apache/nutch/crawl/MD5Signature.java index 8c09543..f0e0332 100644 --- src/java/org/apache/nutch/crawl/MD5Signature.java +++ src/java/org/apache/nutch/crawl/MD5Signature.java @@ -17,14 +17,14 @@ package org.apache.nutch.crawl; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.HashSet; - import org.apache.avro.util.Utf8; import org.apache.hadoop.io.MD5Hash; import org.apache.nutch.storage.WebPage; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; + /** * Default implementation of a page signature. It calculates an MD5 hash * of the raw binary content of a page. In case there is no content, it @@ -47,7 +47,7 @@ public class MD5Signature extends Signature { int of; int cb; if (buf == null) { - Utf8 baseUrl = page.getBaseUrl(); + Utf8 baseUrl = (Utf8) page.getBaseUrl(); if (baseUrl == null) { data = null; of = 0; @@ -56,7 +56,7 @@ public class MD5Signature extends Signature { else { data = baseUrl.getBytes(); of = 0; - cb = baseUrl.getLength(); + cb = baseUrl.length(); } } else { data = buf.array(); diff --git src/java/org/apache/nutch/crawl/WebTableReader.java src/java/org/apache/nutch/crawl/WebTableReader.java index 167f122..e9457f9 100644 --- src/java/org/apache/nutch/crawl/WebTableReader.java +++ src/java/org/apache/nutch/crawl/WebTableReader.java @@ -16,18 +16,11 @@ ******************************************************************************/ package org.apache.nutch.crawl; -import java.io.IOException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; -import java.util.regex.Pattern; - import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.gora.mapreduce.GoraMapper; +import org.apache.gora.query.Query; +import org.apache.gora.query.Result; +import org.apache.gora.store.DataStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -47,17 +40,19 @@ import org.apache.nutch.parse.ParseStatusUtils; import org.apache.nutch.protocol.ProtocolStatusUtils; import org.apache.nutch.storage.StorageUtils; import org.apache.nutch.storage.WebPage; -import org.apache.nutch.util.Bytes; -import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.NutchJob; -import org.apache.nutch.util.NutchTool; -import org.apache.nutch.util.StringUtil; -import org.apache.nutch.util.TableUtil; -import org.apache.nutch.util.ToolUtil; -import org.apache.gora.mapreduce.GoraMapper; -import org.apache.gora.query.Query; -import org.apache.gora.query.Result; -import org.apache.gora.store.DataStore; +import org.apache.nutch.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; +import java.util.regex.Pattern; /** * Displays information about the entries of the webtable @@ -344,7 +339,7 @@ public class WebTableReader extends NutchTool implements Tool { sb.append("key:\t" + key).append("\n"); sb.append("baseUrl:\t" + page.getBaseUrl()).append("\n"); sb.append("status:\t").append(page.getStatus()).append(" (").append( - CrawlStatus.getName((byte) page.getStatus())).append(")\n"); + CrawlStatus.getName(page.getStatus().byteValue())).append(")\n"); sb.append("fetchTime:\t" + page.getFetchTime()).append("\n"); sb.append("prevFetchTime:\t" + page.getPrevFetchTime()).append("\n"); sb.append("fetchInterval:\t" + page.getFetchInterval()).append("\n"); @@ -366,41 +361,41 @@ public class WebTableReader extends NutchTool implements Tool { sb.append("title:\t" + page.getTitle()).append("\n"); sb.append("score:\t" + page.getScore()).append("\n"); - Map markers = page.getMarkers(); + Map markers = page.getMarkers(); sb.append("markers:\t" + markers).append("\n"); sb.append("reprUrl:\t" + page.getReprUrl()).append("\n"); - Utf8 batchId = page.getBatchId(); + CharSequence batchId = page.getBatchId(); if (batchId != null) { sb.append("batchId:\t" + batchId.toString()).append("\n"); } - Map metadata = page.getMetadata(); + Map metadata = page.getMetadata(); if (metadata != null) { - Iterator> iterator = metadata.entrySet() + Iterator> iterator = metadata.entrySet() .iterator(); while (iterator.hasNext()) { - Entry entry = iterator.next(); + Entry entry = iterator.next(); sb.append("metadata " + entry.getKey().toString()).append(" : \t") .append(Bytes.toString(entry.getValue())).append("\n"); } } if (dumpLinks) { - Map inlinks = page.getInlinks(); - Map outlinks = page.getOutlinks(); + Map inlinks = page.getInlinks(); + Map outlinks = page.getOutlinks(); if (outlinks != null) { - for (Entry e : outlinks.entrySet()) { + for (Entry e : outlinks.entrySet()) { sb.append("outlink:\t" + e.getKey() + "\t" + e.getValue() + "\n"); } } if (inlinks != null) { - for (Entry e : inlinks.entrySet()) { + for (Entry e : inlinks.entrySet()) { sb.append("inlink:\t" + e.getKey() + "\t" + e.getValue() + "\n"); } } } if (dumpHeaders) { - Map headers = page.getHeaders(); + Map headers = page.getHeaders(); if (headers != null) { - for (Entry e : headers.entrySet()) { + for (Entry e : headers.entrySet()) { sb.append("header:\t" + e.getKey() + "\t" + e.getValue() + "\n"); } } @@ -412,7 +407,7 @@ public class WebTableReader extends NutchTool implements Tool { sb.append(Bytes.toString(content)); sb.append("\ncontent:end:\n"); } - Utf8 text = page.getText(); + CharSequence text = page.getText(); if (text != null && dumpText) { sb.append("text:start:\n"); sb.append(text.toString()); @@ -521,7 +516,11 @@ public class WebTableReader extends NutchTool implements Tool { DataStore store = StorageUtils.createWebStore(currentJob .getConfiguration(), String.class, WebPage.class); Query query = store.newQuery(); - query.setFields(WebPage._ALL_FIELDS); + + //remove the __g__dirty field since it is not stored + String[] fields = Arrays.copyOfRange(WebPage._ALL_FIELDS, 1, + WebPage._ALL_FIELDS.length); + query.setFields(fields); GoraMapper.initMapperJob(currentJob, query, store, Text.class, LongWritable.class, WebTableStatMapper.class, null, true); diff --git src/java/org/apache/nutch/fetcher/FetcherReducer.java src/java/org/apache/nutch/fetcher/FetcherReducer.java index 055d0fd..069bb45 100644 --- src/java/org/apache/nutch/fetcher/FetcherReducer.java +++ src/java/org/apache/nutch/fetcher/FetcherReducer.java @@ -16,43 +16,19 @@ ******************************************************************************/ package org.apache.nutch.fetcher; -import java.io.IOException; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - +import crawlercommons.robots.BaseRobotRules; import org.apache.avro.util.Utf8; import org.apache.gora.mapreduce.GoraReducer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IntWritable; import org.apache.nutch.crawl.CrawlStatus; -import org.apache.nutch.crawl.URLWebPage; import org.apache.nutch.host.HostDb; import org.apache.nutch.net.URLFilterException; import org.apache.nutch.net.URLFilters; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.parse.ParseUtil; import org.apache.nutch.parse.ParserJob; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.protocol.Protocol; -import org.apache.nutch.protocol.ProtocolFactory; -import org.apache.nutch.protocol.ProtocolOutput; -import org.apache.nutch.protocol.ProtocolStatusCodes; -import org.apache.nutch.protocol.ProtocolStatusUtils; -import org.apache.nutch.protocol.RobotRules; +import org.apache.nutch.protocol.*; import org.apache.nutch.storage.Host; import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.ProtocolStatus; @@ -61,7 +37,14 @@ import org.apache.nutch.util.TableUtil; import org.apache.nutch.util.URLUtil; import org.slf4j.Logger; -import crawlercommons.robots.BaseRobotRules; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; public class FetcherReducer extends GoraReducer { @@ -478,7 +461,7 @@ extends GoraReducer { } } lastRequestStart.set(System.currentTimeMillis()); - if (!fit.page.isReadable(WebPage.Field.REPR_URL.getIndex())) { + if (fit.page.getReprUrl() == null) { reprUrl = fit.url; } else { reprUrl = TableUtil.toString(fit.page.getReprUrl()); @@ -615,8 +598,8 @@ extends GoraReducer { } } - page.putToOutlinks(new Utf8(newUrl), new Utf8()); - page.putToMetadata(FetcherJob.REDIRECT_DISCOVERED, TableUtil.YES_VAL); + page.getOutlinks().put(new Utf8(newUrl), new Utf8()); + page.getMetadata().put(FetcherJob.REDIRECT_DISCOVERED, TableUtil.YES_VAL); reprUrl = URLUtil.chooseRepr(reprUrl, newUrl, temp); if (reprUrl == null) { LOG.warn("reprUrl==null"); @@ -638,7 +621,7 @@ extends GoraReducer { private void output(FetchItem fit, Content content, ProtocolStatus pstatus, byte status) throws IOException, InterruptedException { - fit.page.setStatus(status); + fit.page.setStatus((int)status); final long prevFetchTime = fit.page.getFetchTime(); fit.page.setPrevFetchTime(prevFetchTime); fit.page.setFetchTime(System.currentTimeMillis()); diff --git src/java/org/apache/nutch/host/HostDbUpdateReducer.java src/java/org/apache/nutch/host/HostDbUpdateReducer.java index d471e36..acba4ed 100644 --- src/java/org/apache/nutch/host/HostDbUpdateReducer.java +++ src/java/org/apache/nutch/host/HostDbUpdateReducer.java @@ -16,10 +16,6 @@ ******************************************************************************/ package org.apache.nutch.host; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Set; - import org.apache.avro.util.Utf8; import org.apache.gora.mapreduce.GoraReducer; import org.apache.hadoop.io.Text; @@ -29,6 +25,10 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.Histogram; import org.apache.nutch.util.URLUtil; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Set; + /** * Combines all WebPages with the same host key to create a Host object, * with some statistics. @@ -58,15 +58,15 @@ public class HostDbUpdateReducer extends GoraReducer inlinks = page.getInlinks().keySet(); - for (Utf8 inlink: inlinks) { + Set inlinks = page.getInlinks().keySet(); + for (CharSequence inlink: inlinks) { String host = URLUtil.getHost(inlink.toString()); inlinkCount.add(host); } } if (page.getOutlinks() != null) { - Set outlinks = page.getOutlinks().keySet(); - for (Utf8 outlink: outlinks) { + Set outlinks = page.getOutlinks().keySet(); + for (CharSequence outlink: outlinks) { String host = URLUtil.getHost(outlink.toString()); outlinkCount.add(host); } @@ -76,15 +76,15 @@ public class HostDbUpdateReducer extends GoraReducer 0) { - host.putToMetadata(new Utf8("f"),ByteBuffer.wrap(Integer.toString(numFetched).getBytes())); + host.getMetadata().put(new Utf8("f"),ByteBuffer.wrap(Integer.toString(numFetched).getBytes())); } for (String inlink: inlinkCount.getKeys()) { - host.putToInlinks(new Utf8(inlink), new Utf8(Integer.toString(inlinkCount.getCount(inlink)))); + host.getInlinks().put(new Utf8(inlink), new Utf8(Integer.toString(inlinkCount.getCount(inlink)))); } for (String outlink: outlinkCount.getKeys()) { - host.putToOutlinks(new Utf8(outlink), new Utf8(Integer.toString(outlinkCount.getCount(outlink)))); + host.getInlinks().put(new Utf8(outlink), new Utf8(Integer.toString(outlinkCount.getCount(outlink)))); } context.write(key.toString(), host); diff --git src/java/org/apache/nutch/host/HostInjectorJob.java src/java/org/apache/nutch/host/HostInjectorJob.java index 4910ae8..8780457 100644 --- src/java/org/apache/nutch/host/HostInjectorJob.java +++ src/java/org/apache/nutch/host/HostInjectorJob.java @@ -16,15 +16,6 @@ ******************************************************************************/ package org.apache.nutch.host; -import java.io.IOException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - import org.apache.avro.util.Utf8; import org.apache.gora.mapreduce.GoraOutputFormat; import org.apache.hadoop.conf.Configuration; @@ -46,6 +37,11 @@ import org.apache.nutch.util.TableUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.*; + /** * Creates or updates an existing host table from a text file.
* The files contain one host name per line, optionally followed by custom @@ -127,7 +123,7 @@ public class HostInjectorJob implements Tool { while (keysIter.hasNext()) { String keymd = keysIter.next(); String valuemd = metadata.get(keymd); - host.putToMetadata(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes())); + host.getMetadata().put(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes())); } String hostname; if (url.indexOf("://")> -1) { diff --git src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java index 8e70978..b7bbf97 100644 --- src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java +++ src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java @@ -80,12 +80,12 @@ public class IndexingFiltersChecker extends Configured implements Tool { ProtocolFactory factory = new ProtocolFactory(conf); Protocol protocol = factory.getProtocol(url); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new org.apache.avro.util.Utf8(url)); ProtocolOutput protocolOutput = protocol.getProtocolOutput(url, page); page.setProtocolStatus(protocolOutput.getStatus()); if (protocolOutput.getStatus().getCode() == ProtocolStatusCodes.SUCCESS) { - page.setStatus(CrawlStatus.STATUS_FETCHED); + page.setStatus((int)CrawlStatus.STATUS_FETCHED); page.setFetchTime(System.currentTimeMillis()); } else { LOG.error("Fetch failed with protocol status: " diff --git src/java/org/apache/nutch/parse/ParseStatusUtils.java src/java/org/apache/nutch/parse/ParseStatusUtils.java index 179e85e..f642980 100644 --- src/java/org/apache/nutch/parse/ParseStatusUtils.java +++ src/java/org/apache/nutch/parse/ParseStatusUtils.java @@ -16,22 +16,23 @@ ******************************************************************************/ package org.apache.nutch.parse; -import java.util.HashMap; -import java.util.Iterator; - import org.apache.avro.generic.GenericArray; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.storage.ParseStatus; import org.apache.nutch.util.TableUtil; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + public class ParseStatusUtils { - public static ParseStatus STATUS_SUCCESS = new ParseStatus(); + public static ParseStatus STATUS_SUCCESS = ParseStatus.newBuilder().build(); public static final HashMap minorCodes = new HashMap(); static { - STATUS_SUCCESS.setMajorCode(ParseStatusCodes.SUCCESS); + STATUS_SUCCESS.setMajorCode((int)ParseStatusCodes.SUCCESS); minorCodes.put(ParseStatusCodes.SUCCESS_OK, "ok"); minorCodes.put(ParseStatusCodes.SUCCESS_REDIRECT, "redirect"); minorCodes.put(ParseStatusCodes.FAILED_EXCEPTION, "exception"); @@ -52,7 +53,7 @@ public class ParseStatusUtils { * argument, or null. */ public static String getMessage(ParseStatus status) { - GenericArray args = status.getArgs(); + List args = status.getArgs(); if (args != null && args.size() > 0) { return TableUtil.toString(args.iterator().next()); } @@ -60,12 +61,12 @@ public class ParseStatusUtils { } public static String getArg(ParseStatus status, int n) { - GenericArray args = status.getArgs(); + List args = status.getArgs(); if (args == null) { return null; } int i = 0; - for (Utf8 arg : args) { + for (CharSequence arg : args) { if (i == n) { return TableUtil.toString(arg); } @@ -75,19 +76,19 @@ public class ParseStatusUtils { } public static Parse getEmptyParse(Exception e, Configuration conf) { - ParseStatus status = new ParseStatus(); - status.setMajorCode(ParseStatusCodes.FAILED); - status.setMinorCode(ParseStatusCodes.FAILED_EXCEPTION); - status.addToArgs(new Utf8(e.toString())); + ParseStatus status = ParseStatus.newBuilder().build(); + status.setMajorCode((int)ParseStatusCodes.FAILED); + status.setMinorCode((int)ParseStatusCodes.FAILED_EXCEPTION); + status.getArgs().add(new Utf8(e.toString())); return new Parse("", "", new Outlink[0], status); } public static Parse getEmptyParse(int minorCode, String message, Configuration conf) { ParseStatus status = new ParseStatus(); - status.setMajorCode(ParseStatusCodes.FAILED); + status.setMajorCode((int)ParseStatusCodes.FAILED); status.setMinorCode(minorCode); - status.addToArgs(new Utf8(message)); + status.getArgs().add(new Utf8(message)); return new Parse("", "", new Outlink[0], status); } @@ -98,13 +99,13 @@ public class ParseStatusUtils { } StringBuilder sb = new StringBuilder(); sb.append(ParseStatusCodes.majorCodes[status.getMajorCode()] + - "/" + minorCodes.get((short)status.getMinorCode())); + "/" + minorCodes.get(status.getMinorCode().shortValue())); sb.append(" (" + status.getMajorCode() + "/" + status.getMinorCode() + ")"); sb.append(", args=["); - GenericArray args = status.getArgs(); + List args = status.getArgs(); if (args != null) { int i = 0; - Iterator it = args.iterator(); + Iterator it = args.iterator(); while (it.hasNext()) { if (i > 0) sb.append(','); sb.append(it.next()); diff --git src/java/org/apache/nutch/parse/ParseUtil.java src/java/org/apache/nutch/parse/ParseUtil.java index 04121a6..e517315 100644 --- src/java/org/apache/nutch/parse/ParseUtil.java +++ src/java/org/apache/nutch/parse/ParseUtil.java @@ -17,14 +17,8 @@ package org.apache.nutch.parse; // Commons Logging imports -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -43,7 +37,13 @@ import org.apache.nutch.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; /** * A Utility class containing methods to simply perform parsing utilities such @@ -162,7 +162,7 @@ public class ParseUtil extends Configured { */ public void process(String key, WebPage page) { String url = TableUtil.unreverseUrl(key); - byte status = (byte) page.getStatus(); + byte status = page.getStatus().byteValue(); if (status != CrawlStatus.STATUS_FETCHED) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping " + url + " as status is: " + CrawlStatus.getName(status)); @@ -214,8 +214,8 @@ public class ParseUtil extends Configured { LOG.warn("malformed url exception parsing redirect " + url); return; } - page.putToOutlinks(new Utf8(newUrl), new Utf8()); - page.putToMetadata(FetcherJob.REDIRECT_DISCOVERED, TableUtil.YES_VAL); + page.getOutlinks().put(new Utf8(newUrl), new Utf8()); + page.getMetadata().put(FetcherJob.REDIRECT_DISCOVERED, TableUtil.YES_VAL); if (newUrl == null || newUrl.equals(url)) { String reprUrl = URLUtil.chooseRepr(url, newUrl, refreshTime < FetcherJob.PERM_REFRESH_TIME); @@ -265,7 +265,7 @@ public class ParseUtil extends Configured { continue; } Utf8 utf8ToUrl = new Utf8(toUrl); - if (page.getFromOutlinks(utf8ToUrl) != null) { + if (page.getOutlinks().get(utf8ToUrl) != null) { // skip duplicate outlinks continue; } @@ -281,7 +281,7 @@ public class ParseUtil extends Configured { } } validCount++; - page.putToOutlinks(utf8ToUrl, new Utf8(outlinks[i].getAnchor())); + page.getOutlinks().put(utf8ToUrl, new Utf8(outlinks[i].getAnchor())); } Utf8 fetchMark = Mark.FETCH_MARK.checkMark(page); if (fetchMark != null) { diff --git src/java/org/apache/nutch/parse/ParserChecker.java src/java/org/apache/nutch/parse/ParserChecker.java index 53f0e83..c3aac32 100644 --- src/java/org/apache/nutch/parse/ParserChecker.java +++ src/java/org/apache/nutch/parse/ParserChecker.java @@ -17,28 +17,24 @@ package org.apache.nutch.parse; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; - import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.nutch.crawl.SignatureFactory; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.protocol.Protocol; -import org.apache.nutch.protocol.ProtocolFactory; -import org.apache.nutch.protocol.ProtocolOutput; -import org.apache.nutch.protocol.ProtocolStatusUtils; +import org.apache.nutch.protocol.*; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.Bytes; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.StringUtil; import org.apache.nutch.util.URLUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; /** * Parser checker, useful for testing parser. @@ -107,7 +103,7 @@ public class ParserChecker implements Tool { ProtocolFactory factory = new ProtocolFactory(conf); Protocol protocol = factory.getProtocol(url); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); ProtocolOutput protocolOutput = protocol.getProtocolOutput(url, page); @@ -163,13 +159,13 @@ public class ParserChecker implements Tool { LOG.info("---------\nUrl\n---------------\n"); System.out.print(url + "\n"); LOG.info("---------\nMetadata\n---------\n"); - Map metadata = page.getMetadata(); + Map metadata = page.getMetadata(); StringBuffer sb = new StringBuffer(); if (metadata != null) { - Iterator> iterator = metadata.entrySet() + Iterator> iterator = metadata.entrySet() .iterator(); while (iterator.hasNext()) { - Entry entry = iterator.next(); + Entry entry = iterator.next(); sb.append(entry.getKey().toString()).append(" : \t") .append(Bytes.toString(entry.getValue())).append("\n"); } diff --git src/java/org/apache/nutch/parse/ParserJob.java src/java/org/apache/nutch/parse/ParserJob.java index 9f8ce8f..ba17744 100644 --- src/java/org/apache/nutch/parse/ParserJob.java +++ src/java/org/apache/nutch/parse/ParserJob.java @@ -102,7 +102,7 @@ public class ParserJob extends NutchTool implements Tool { @Override public void map(String key, WebPage page, Context context) throws IOException, InterruptedException { - Utf8 mark = Mark.FETCH_MARK.checkMark(page); + CharSequence mark = Mark.FETCH_MARK.checkMark(page); String unreverseKey = TableUtil.unreverseUrl(key); if (batchId.equals(REPARSE)) { LOG.debug("Reparsing " + unreverseKey); @@ -161,7 +161,7 @@ public class ParserJob extends NutchTool implements Tool { if (content == null) { return false; } - Utf8 lengthUtf8 = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_LENGTH)); + CharSequence lengthUtf8 = page.getHeaders().get(new Utf8(HttpHeaders.CONTENT_LENGTH)); if (lengthUtf8 == null) { return false; } diff --git src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java index 438c967..7f1f11c 100644 --- src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java +++ src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java @@ -16,14 +16,15 @@ ******************************************************************************/ package org.apache.nutch.protocol; -import java.net.URL; -import java.util.Iterator; - import org.apache.avro.generic.GenericArray; import org.apache.avro.util.Utf8; import org.apache.nutch.storage.ProtocolStatus; import org.apache.nutch.util.TableUtil; +import java.net.URL; +import java.util.Iterator; +import java.util.List; + public class ProtocolStatusUtils implements ProtocolStatusCodes { // Useful static instances for status codes that don't usually require any // additional arguments. @@ -76,15 +77,15 @@ public class ProtocolStatusUtils implements ProtocolStatusCodes { } public static ProtocolStatus makeStatus(int code) { - ProtocolStatus pstatus = new ProtocolStatus(); + ProtocolStatus pstatus = ProtocolStatus.newBuilder().build(); pstatus.setCode(code); - pstatus.setLastModified(0); + pstatus.setLastModified(0L); return pstatus; } public static ProtocolStatus makeStatus(int code, String message) { ProtocolStatus pstatus = makeStatus(code); - pstatus.addToArgs(new Utf8(message)); + pstatus.getArgs().add(new Utf8(message)); return pstatus; } @@ -93,7 +94,7 @@ public class ProtocolStatusUtils implements ProtocolStatusCodes { } public static String getMessage(ProtocolStatus pstatus) { - GenericArray args = pstatus.getArgs(); + List args = pstatus.getArgs(); if (args == null || args.size() == 0) { return null; } @@ -107,10 +108,10 @@ public class ProtocolStatusUtils implements ProtocolStatusCodes { StringBuilder sb = new StringBuilder(); sb.append(getName(status.getCode())); sb.append(", args=["); - GenericArray args = status.getArgs(); + List args = status.getArgs(); if (args != null) { int i = 0; - Iterator it = args.iterator(); + Iterator it = args.iterator(); while (it.hasNext()) { if (i > 0) sb.append(','); sb.append(it.next()); diff --git src/java/org/apache/nutch/storage/Host.java src/java/org/apache/nutch/storage/Host.java index 941415f..b577344 100644 --- src/java/org/apache/nutch/storage/Host.java +++ src/java/org/apache/nutch/storage/Host.java @@ -1,157 +1,444 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Autogenerated by Avro * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ + * DO NOT EDIT DIRECTLY + */ package org.apache.nutch.storage; -import java.nio.ByteBuffer; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; import org.apache.avro.util.Utf8; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; -import org.apache.gora.persistency.StatefulHashMap; import org.apache.nutch.util.Bytes; @SuppressWarnings("all") -public class Host extends PersistentBase { - public static final org.apache.avro.Schema _SCHEMA = org.apache.avro.Schema.parse("{\"type\":\"record\",\"name\":\"Host\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}]}"); - public java.util.Map metadata; - public java.util.Map outlinks; - public java.util.Map inlinks; - +public class Host extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Host\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"},\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}"); + + /** Enum containing all data bean's fields. */ public static enum Field { - METADATA(0,"metadata"), - OUTLINKS(1,"outlinks"), - INLINKS(2,"inlinks"), + __G__DIRTY(0, "__g__dirty"), + METADATA(1, "metadata"), + OUTLINKS(2, "outlinks"), + INLINKS(3, "inlinks"), ; + /** + * Field's index. + */ private int index; + + /** + * Field's name. + */ private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"metadata","outlinks","inlinks"}; - static { - PersistentBase.registerFields(Host.class, _ALL_FIELDS); + + public static final String[] _ALL_FIELDS = { + "__g__dirty", + "metadata", + "outlinks", + "inlinks", + }; + + /** Bytes used to represent weather or not a field is dirty. */ + private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]); + private java.util.Map metadata; + private java.util.Map outlinks; + private java.util.Map inlinks; + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public Object get(int field$) { + switch (field$) { + case 0: return __g__dirty; + case 1: return metadata; + case 2: return outlinks; + case 3: return inlinks; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, Object value) { + switch (field$) { + case 0: __g__dirty = (java.nio.ByteBuffer)(value); break; + case 1: metadata = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 2: outlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 3: inlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } } - public Host() { - this(new StateManagerImpl()); + /** + * Gets the value of the 'metadata' field. + */ + public java.util.Map getMetadata() { + return metadata; } - public Host(StateManager stateManager) { - super(stateManager); - metadata = new StatefulHashMap(); - inlinks = new StatefulHashMap(); - outlinks = new StatefulHashMap(); + + /** + * Sets the value of the 'metadata' field. + * @param value the value to set. + */ + public void setMetadata(java.util.Map value) { + this.metadata = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(1); } - public Host newInstance(StateManager stateManager) { - return new Host(stateManager); + + /** + * Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMetadataDirty(java.util.Map value) { + return isDirty(1); } - public Schema getSchema() { return _SCHEMA; } - public Object get(int _field) { - switch (_field) { - case 0: return metadata; - case 1: return outlinks; - case 2: return inlinks; - default: throw new AvroRuntimeException("Bad index"); - } + + /** + * Gets the value of the 'outlinks' field. + */ + public java.util.Map getOutlinks() { + return outlinks; } - @SuppressWarnings(value="unchecked") - public void put(int _field, Object _value) { - - if(isFieldEqual(_field, _value)) return; - getStateManager().setDirty(this, _field); - switch (_field) { - case 0: metadata = (Map)_value; break; - case 1: outlinks = (Map)_value; break; - case 2: inlinks = (Map)_value; break; - default: throw new AvroRuntimeException("Bad index"); - } - } - @SuppressWarnings("unchecked") - public Map getMetadata() { - return (Map) get(0); - } - public ByteBuffer getFromMetadata(Utf8 key) { - if (metadata == null) { return null; } - return metadata.get(key); + + /** + * Sets the value of the 'outlinks' field. + * @param value the value to set. + */ + public void setOutlinks(java.util.Map value) { + this.outlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(2); } - public void putToMetadata(Utf8 key, ByteBuffer value) { - getStateManager().setDirty(this, 0); - metadata.put(key, value); + /** + * Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isOutlinksDirty(java.util.Map value) { + return isDirty(2); } - public ByteBuffer removeFromMetadata(Utf8 key) { - if (metadata == null) { return null; } - getStateManager().setDirty(this, 0); - return metadata.remove(key); + + /** + * Gets the value of the 'inlinks' field. + */ + public java.util.Map getInlinks() { + return inlinks; } - @SuppressWarnings("unchecked") - public Map getOutlinks() { - return (Map) get(1); + + /** + * Sets the value of the 'inlinks' field. + * @param value the value to set. + */ + public void setInlinks(java.util.Map value) { + this.inlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(3); } - public Utf8 getFromOutlinks(Utf8 key) { - if (outlinks == null) { return null; } - return outlinks.get(key); + + /** + * Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isInlinksDirty(java.util.Map value) { + return isDirty(3); } - public void putToOutlinks(Utf8 key, Utf8 value) { - getStateManager().setDirty(this, 1); - outlinks.put(key, value); + + public boolean contains(String key) { + return metadata.containsKey(new Utf8(key)); } - public Utf8 removeFromOutlinks(Utf8 key) { - if (outlinks == null) { return null; } - getStateManager().setDirty(this, 1); - return outlinks.remove(key); + + public String getValue(String key, String defaultValue) { + if (!contains(key)) + return defaultValue; + return Bytes.toString(metadata.get(new Utf8(key))); } - @SuppressWarnings("unchecked") - public Map getInlinks() { - return (Map) get(2); + + public int getInt(String key, int defaultValue) { + if (!contains(key)) + return defaultValue; + return Integer.parseInt(getValue(key, null)); } - public Utf8 getFromInlinks(Utf8 key) { - if (inlinks == null) { return null; } - return inlinks.get(key); + + public long getLong(String key, long defaultValue) { + if (!contains(key)) + return defaultValue; + return Long.parseLong(getValue(key, null)); } - public void putToInlinks(Utf8 key, Utf8 value) { - getStateManager().setDirty(this, 2); - inlinks.put(key, value); + + /** Creates a new Host RecordBuilder */ + public static Builder newBuilder() { + return new Builder(); } - public Utf8 removeFromInlinks(Utf8 key) { - if (inlinks == null) { return null; } - getStateManager().setDirty(this, 2); - return inlinks.remove(key); + + /** Creates a new Host RecordBuilder by copying an existing Builder */ + public static Builder newBuilder(Builder other) { + return new Builder(other); } - public boolean contains(String key) { - return metadata.containsKey(new Utf8(key)); + /** Creates a new Host RecordBuilder by copying an existing Host instance */ + public static Builder newBuilder(Host other) { + return new Builder(other); } - public String getValue(String key, String defaultValue) { - if (!contains(key)) return defaultValue; - return Bytes.toString(metadata.get(new Utf8(key))); + private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); } - public int getInt(String key, int defaultValue) { - if (!contains(key)) return defaultValue; - return Integer.parseInt(getValue(key,null)); + /** + * RecordBuilder for Host instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.nio.ByteBuffer __g__dirty; + private java.util.Map metadata; + private java.util.Map outlinks; + private java.util.Map inlinks; + + /** Creates a new Builder */ + private Builder() { + super(Host.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing Host instance */ + private Builder(Host other) { + super(Host.SCHEMA$); + if (isValidValue(fields()[0], other.__g__dirty)) { + this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.metadata)) { + this.metadata = (java.util.Map) data().deepCopy(fields()[1].schema(), other.metadata); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.outlinks)) { + this.outlinks = (java.util.Map) data().deepCopy(fields()[2].schema(), other.outlinks); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.inlinks)) { + this.inlinks = (java.util.Map) data().deepCopy(fields()[3].schema(), other.inlinks); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'metadata' field */ + public java.util.Map getMetadata() { + return metadata; + } + + /** Sets the value of the 'metadata' field */ + public Builder setMetadata(java.util.Map value) { + validate(fields()[1], value); + this.metadata = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'metadata' field has been set */ + public boolean hasMetadata() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'metadata' field */ + public Builder clearMetadata() { + metadata = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'outlinks' field */ + public java.util.Map getOutlinks() { + return outlinks; + } + + /** Sets the value of the 'outlinks' field */ + public Builder setOutlinks(java.util.Map value) { + validate(fields()[2], value); + this.outlinks = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'outlinks' field has been set */ + public boolean hasOutlinks() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'outlinks' field */ + public Builder clearOutlinks() { + outlinks = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'inlinks' field */ + public java.util.Map getInlinks() { + return inlinks; + } + + /** Sets the value of the 'inlinks' field */ + public Builder setInlinks(java.util.Map value) { + validate(fields()[3], value); + this.inlinks = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'inlinks' field has been set */ + public boolean hasInlinks() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'inlinks' field */ + public Builder clearInlinks() { + inlinks = null; + fieldSetFlags()[3] = false; + return this; + } + + @Override + public Host build() { + try { + Host record = new Host(); + record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]); + record.metadata = fieldSetFlags()[1] ? this.metadata : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[1])); + record.outlinks = fieldSetFlags()[2] ? this.outlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[2])); + record.inlinks = fieldSetFlags()[3] ? this.inlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[3])); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } } - public long getLong(String key, long defaultValue) { - if (!contains(key)) return defaultValue; - return Long.parseLong(getValue(key,null)); + + public Tombstone getTombstone(){ + return TOMBSTONE; + } + + public Host newInstance(){ + return newBuilder().build(); + } + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends Host implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'metadata' field. + */ + public java.util.Map getMetadata() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'metadata' field. + * @param value the value to set. + */ + public void setMetadata(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMetadataDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'outlinks' field. + */ + public java.util.Map getOutlinks() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'outlinks' field. + * @param value the value to set. + */ + public void setOutlinks(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isOutlinksDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'inlinks' field. + */ + public java.util.Map getInlinks() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'inlinks' field. + * @param value the value to set. + */ + public void setInlinks(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isInlinksDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + } -} + +} \ No newline at end of file diff --git src/java/org/apache/nutch/storage/Mark.java src/java/org/apache/nutch/storage/Mark.java index f20ed9c..d334304 100644 --- src/java/org/apache/nutch/storage/Mark.java +++ src/java/org/apache/nutch/storage/Mark.java @@ -29,7 +29,7 @@ public enum Mark { } public void putMark(WebPage page, Utf8 markValue) { - page.putToMarkers(name, markValue); + page.getMarkers().put(name, markValue); } public void putMark(WebPage page, String markValue) { @@ -37,11 +37,11 @@ public enum Mark { } public Utf8 removeMark(WebPage page) { - return page.removeFromMarkers(name); + return (Utf8) page.getMarkers().put(name, null); } public Utf8 checkMark(WebPage page) { - return page.getFromMarkers(name); + return (Utf8) page.getMarkers().get(name); } /** @@ -50,8 +50,8 @@ public enum Mark { * @return If the mark was present. */ public Utf8 removeMarkIfExist(WebPage page) { - if (page.getFromMarkers(name) != null) { - return page.removeFromMarkers(name); + if (checkMark(page) != null) { + return removeMark(page); } return null; } diff --git src/java/org/apache/nutch/storage/ParseStatus.java src/java/org/apache/nutch/storage/ParseStatus.java index 68dc651..60e8afc 100644 --- src/java/org/apache/nutch/storage/ParseStatus.java +++ src/java/org/apache/nutch/storage/ParseStatus.java @@ -1,113 +1,416 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Autogenerated by Avro * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.nutch.storage; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import org.apache.avro.Protocol; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Protocol; -import org.apache.avro.util.Utf8; -import org.apache.avro.ipc.AvroRemoteException; -import org.apache.avro.generic.GenericArray; -import org.apache.avro.specific.FixedSize; -import org.apache.avro.specific.SpecificExceptionBase; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.specific.SpecificFixed; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; -import org.apache.gora.persistency.StatefulHashMap; -import org.apache.gora.persistency.ListGenericArray; - + * DO NOT EDIT DIRECTLY + */ +package org.apache.nutch.storage; @SuppressWarnings("all") -public class ParseStatus extends PersistentBase { - public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\"},{\"name\":\"minorCode\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}"); +public class ParseStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"majorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}"); + + /** Enum containing all data bean's fields. */ public static enum Field { - MAJOR_CODE(0,"majorCode"), - MINOR_CODE(1,"minorCode"), - ARGS(2,"args"), + __G__DIRTY(0, "__g__dirty"), + MAJOR_CODE(1, "majorCode"), + MINOR_CODE(2, "minorCode"), + ARGS(3, "args"), ; + /** + * Field's index. + */ private int index; + + /** + * Field's name. + */ private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"majorCode","minorCode","args",}; - static { - PersistentBase.registerFields(ParseStatus.class, _ALL_FIELDS); - } + + public static final String[] _ALL_FIELDS = { + "__g__dirty", + "majorCode", + "minorCode", + "args", + }; + + /** Bytes used to represent weather or not a field is dirty. */ + private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]); private int majorCode; private int minorCode; - private GenericArray args; - public ParseStatus() { - this(new StateManagerImpl()); - } - public ParseStatus(StateManager stateManager) { - super(stateManager); - args = new ListGenericArray(getSchema().getField("args").schema()); - } - public ParseStatus newInstance(StateManager stateManager) { - return new ParseStatus(stateManager); - } - public Schema getSchema() { return _SCHEMA; } - public Object get(int _field) { - switch (_field) { - case 0: return majorCode; - case 1: return minorCode; - case 2: return args; - default: throw new AvroRuntimeException("Bad index"); + private java.util.List args; + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public Object get(int field$) { + switch (field$) { + case 0: return __g__dirty; + case 1: return majorCode; + case 2: return minorCode; + case 3: return args; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } + + // Used by DatumReader. Applications should not call. @SuppressWarnings(value="unchecked") - public void put(int _field, Object _value) { - if(isFieldEqual(_field, _value)) return; - getStateManager().setDirty(this, _field); - switch (_field) { - case 0:majorCode = (Integer)_value; break; - case 1:minorCode = (Integer)_value; break; - case 2:args = (GenericArray)_value; break; - default: throw new AvroRuntimeException("Bad index"); + public void put(int field$, Object value) { + switch (field$) { + case 0: __g__dirty = (java.nio.ByteBuffer)(value); break; + case 1: majorCode = (Integer)(value); break; + case 2: minorCode = (Integer)(value); break; + case 3: args = (java.util.List)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } - public int getMajorCode() { - return (Integer) get(0); + + /** + * Gets the value of the 'majorCode' field. + */ + public Integer getMajorCode() { + return majorCode; + } + + /** + * Sets the value of the 'majorCode' field. + * @param value the value to set. + */ + public void setMajorCode(Integer value) { + this.majorCode = value; + setDirty(1); + } + + /** + * Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMajorCodeDirty(Integer value) { + return isDirty(1); + } + + /** + * Gets the value of the 'minorCode' field. + */ + public Integer getMinorCode() { + return minorCode; + } + + /** + * Sets the value of the 'minorCode' field. + * @param value the value to set. + */ + public void setMinorCode(Integer value) { + this.minorCode = value; + setDirty(2); + } + + /** + * Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMinorCodeDirty(Integer value) { + return isDirty(2); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List getArgs() { + return args; + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List value) { + this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value); + setDirty(3); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List value) { + return isDirty(3); } - public void setMajorCode(int value) { - put(0, value); + + /** Creates a new ParseStatus RecordBuilder */ + public static Builder newBuilder() { + return new Builder(); } - public int getMinorCode() { - return (Integer) get(1); + + /** Creates a new ParseStatus RecordBuilder by copying an existing Builder */ + public static Builder newBuilder(Builder other) { + return new Builder(other); } - public void setMinorCode(int value) { - put(1, value); + + /** Creates a new ParseStatus RecordBuilder by copying an existing ParseStatus instance */ + public static Builder newBuilder(ParseStatus other) { + return new Builder(other); } - @SuppressWarnings("unchecked") - public GenericArray getArgs() { - return (GenericArray) get(2); + + private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); } - public void addToArgs(Utf8 element) { - getStateManager().setDirty(this, 2); - args.add(element); + + /** + * RecordBuilder for ParseStatus instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.nio.ByteBuffer __g__dirty; + private int majorCode; + private int minorCode; + private java.util.List args; + + /** Creates a new Builder */ + private Builder() { + super(ParseStatus.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing ParseStatus instance */ + private Builder(ParseStatus other) { + super(ParseStatus.SCHEMA$); + if (isValidValue(fields()[0], other.__g__dirty)) { + this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.majorCode)) { + this.majorCode = (Integer) data().deepCopy(fields()[1].schema(), other.majorCode); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.minorCode)) { + this.minorCode = (Integer) data().deepCopy(fields()[2].schema(), other.minorCode); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.args)) { + this.args = (java.util.List) data().deepCopy(fields()[3].schema(), other.args); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'majorCode' field */ + public Integer getMajorCode() { + return majorCode; + } + + /** Sets the value of the 'majorCode' field */ + public Builder setMajorCode(int value) { + validate(fields()[1], value); + this.majorCode = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'majorCode' field has been set */ + public boolean hasMajorCode() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'majorCode' field */ + public Builder clearMajorCode() { + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'minorCode' field */ + public Integer getMinorCode() { + return minorCode; + } + + /** Sets the value of the 'minorCode' field */ + public Builder setMinorCode(int value) { + validate(fields()[2], value); + this.minorCode = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'minorCode' field has been set */ + public boolean hasMinorCode() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'minorCode' field */ + public Builder clearMinorCode() { + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'args' field */ + public java.util.List getArgs() { + return args; + } + + /** Sets the value of the 'args' field */ + public Builder setArgs(java.util.List value) { + validate(fields()[3], value); + this.args = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'args' field has been set */ + public boolean hasArgs() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'args' field */ + public Builder clearArgs() { + args = null; + fieldSetFlags()[3] = false; + return this; + } + + @Override + public ParseStatus build() { + try { + ParseStatus record = new ParseStatus(); + record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]); + record.majorCode = fieldSetFlags()[1] ? this.majorCode : (Integer) defaultValue(fields()[1]); + record.minorCode = fieldSetFlags()[2] ? this.minorCode : (Integer) defaultValue(fields()[2]); + record.args = fieldSetFlags()[3] ? this.args : (java.util.List) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[3])); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + public Tombstone getTombstone(){ + return TOMBSTONE; + } + + public ParseStatus newInstance(){ + return newBuilder().build(); + } + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends ParseStatus implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'majorCode' field. + */ + public Integer getMajorCode() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'majorCode' field. + * @param value the value to set. + */ + public void setMajorCode(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMajorCodeDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'minorCode' field. + */ + public Integer getMinorCode() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'minorCode' field. + * @param value the value to set. + */ + public void setMinorCode(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMinorCodeDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List getArgs() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + } -} + +} \ No newline at end of file diff --git src/java/org/apache/nutch/storage/ProtocolStatus.java src/java/org/apache/nutch/storage/ProtocolStatus.java index b0845b6..d8acaa9 100644 --- src/java/org/apache/nutch/storage/ProtocolStatus.java +++ src/java/org/apache/nutch/storage/ProtocolStatus.java @@ -1,122 +1,428 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Autogenerated by Avro * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.nutch.storage; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import org.apache.avro.Protocol; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Protocol; -import org.apache.avro.util.Utf8; -import org.apache.avro.ipc.AvroRemoteException; -import org.apache.avro.generic.GenericArray; -import org.apache.avro.specific.FixedSize; -import org.apache.avro.specific.SpecificExceptionBase; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.specific.SpecificFixed; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; -import org.apache.gora.persistency.StatefulHashMap; -import org.apache.gora.persistency.ListGenericArray; + * DO NOT EDIT DIRECTLY + */ +package org.apache.nutch.storage; + import org.apache.nutch.protocol.ProtocolStatusUtils; @SuppressWarnings("all") -public class ProtocolStatus extends PersistentBase { - public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"code\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"lastModified\",\"type\":\"long\"}]}"); +public class ProtocolStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"code\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"default\":0}]}"); + + /** Enum containing all data bean's fields. */ public static enum Field { - CODE(0,"code"), - ARGS(1,"args"), - LAST_MODIFIED(2,"lastModified"), + __G__DIRTY(0, "__g__dirty"), + CODE(1, "code"), + ARGS(2, "args"), + LAST_MODIFIED(3, "lastModified"), ; + /** + * Field's index. + */ private int index; + + /** + * Field's name. + */ private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"code","args","lastModified",}; - static { - PersistentBase.registerFields(ProtocolStatus.class, _ALL_FIELDS); - } + + public static final String[] _ALL_FIELDS = { + "__g__dirty", + "code", + "args", + "lastModified", + }; + + /** Bytes used to represent weather or not a field is dirty. */ + private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]); private int code; - private GenericArray args; + private java.util.List args; private long lastModified; - public ProtocolStatus() { - this(new StateManagerImpl()); - } - public ProtocolStatus(StateManager stateManager) { - super(stateManager); - args = new ListGenericArray(getSchema().getField("args").schema()); - } - public ProtocolStatus newInstance(StateManager stateManager) { - return new ProtocolStatus(stateManager); - } - public Schema getSchema() { return _SCHEMA; } - public Object get(int _field) { - switch (_field) { - case 0: return code; - case 1: return args; - case 2: return lastModified; - default: throw new AvroRuntimeException("Bad index"); + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public Object get(int field$) { + switch (field$) { + case 0: return __g__dirty; + case 1: return code; + case 2: return args; + case 3: return lastModified; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } + + // Used by DatumReader. Applications should not call. @SuppressWarnings(value="unchecked") - public void put(int _field, Object _value) { - if(isFieldEqual(_field, _value)) return; - getStateManager().setDirty(this, _field); - switch (_field) { - case 0:code = (Integer)_value; break; - case 1:args = (GenericArray)_value; break; - case 2:lastModified = (Long)_value; break; - default: throw new AvroRuntimeException("Bad index"); + public void put(int field$, Object value) { + switch (field$) { + case 0: __g__dirty = (java.nio.ByteBuffer)(value); break; + case 1: code = (Integer)(value); break; + case 2: args = (java.util.List)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break; + case 3: lastModified = (Long)(value); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } - public int getCode() { - return (Integer) get(0); + + /** + * Gets the value of the 'code' field. + */ + public Integer getCode() { + return code; + } + + /** + * Sets the value of the 'code' field. + * @param value the value to set. + */ + public void setCode(Integer value) { + this.code = value; + setDirty(1); + } + + /** + * Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isCodeDirty(Integer value) { + return isDirty(1); } - public void setCode(int value) { - put(0, value); + + /** + * Gets the value of the 'args' field. + */ + public java.util.List getArgs() { + return args; } - @SuppressWarnings("unchecked") - public GenericArray getArgs() { - return (GenericArray) get(1); + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List value) { + this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value); + setDirty(2); } - public void addToArgs(Utf8 element) { - getStateManager().setDirty(this, 1); - args.add(element); + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List value) { + return isDirty(2); } - public long getLastModified() { - return (Long) get(2); + + /** + * Gets the value of the 'lastModified' field. + */ + public Long getLastModified() { + return lastModified; } - public void setLastModified(long value) { - put(2, value); + + /** + * Sets the value of the 'lastModified' field. + * @param value the value to set. + */ + public void setLastModified(Long value) { + this.lastModified = value; + setDirty(3); } /** + * Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isLastModifiedDirty(Long value) { + return isDirty(3); + } + + /** * A convenience method which returns a successful {@link ProtocolStatus}. + * * @return the {@link ProtocolStatus} value for 200 (success). */ public boolean isSuccess() { - return code == ProtocolStatusUtils.SUCCESS; + return code == ProtocolStatusUtils.SUCCESS; + } + + /** Creates a new ProtocolStatus RecordBuilder */ + public static Builder newBuilder() { + return new Builder(); + } + + /** Creates a new ProtocolStatus RecordBuilder by copying an existing Builder */ + public static Builder newBuilder(Builder other) { + return new Builder(other); } -} + + /** Creates a new ProtocolStatus RecordBuilder by copying an existing ProtocolStatus instance */ + public static Builder newBuilder(ProtocolStatus other) { + return new Builder(other); + } + + private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); + } + + /** + * RecordBuilder for ProtocolStatus instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.nio.ByteBuffer __g__dirty; + private int code; + private java.util.List args; + private long lastModified; + + /** Creates a new Builder */ + private Builder() { + super(ProtocolStatus.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing ProtocolStatus instance */ + private Builder(ProtocolStatus other) { + super(ProtocolStatus.SCHEMA$); + if (isValidValue(fields()[0], other.__g__dirty)) { + this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.code)) { + this.code = (Integer) data().deepCopy(fields()[1].schema(), other.code); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.args)) { + this.args = (java.util.List) data().deepCopy(fields()[2].schema(), other.args); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.lastModified)) { + this.lastModified = (Long) data().deepCopy(fields()[3].schema(), other.lastModified); + fieldSetFlags()[3] = true; + } + } + + /** Gets the value of the 'code' field */ + public Integer getCode() { + return code; + } + + /** Sets the value of the 'code' field */ + public Builder setCode(int value) { + validate(fields()[1], value); + this.code = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'code' field has been set */ + public boolean hasCode() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'code' field */ + public Builder clearCode() { + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'args' field */ + public java.util.List getArgs() { + return args; + } + + /** Sets the value of the 'args' field */ + public Builder setArgs(java.util.List value) { + validate(fields()[2], value); + this.args = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'args' field has been set */ + public boolean hasArgs() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'args' field */ + public Builder clearArgs() { + args = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'lastModified' field */ + public Long getLastModified() { + return lastModified; + } + + /** Sets the value of the 'lastModified' field */ + public Builder setLastModified(long value) { + validate(fields()[3], value); + this.lastModified = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'lastModified' field has been set */ + public boolean hasLastModified() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'lastModified' field */ + public Builder clearLastModified() { + fieldSetFlags()[3] = false; + return this; + } + + @Override + public ProtocolStatus build() { + try { + ProtocolStatus record = new ProtocolStatus(); + record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]); + record.code = fieldSetFlags()[1] ? this.code : (Integer) defaultValue(fields()[1]); + record.args = fieldSetFlags()[2] ? this.args : (java.util.List) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[2])); + record.lastModified = fieldSetFlags()[3] ? this.lastModified : (Long) defaultValue(fields()[3]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + public Tombstone getTombstone(){ + return TOMBSTONE; + } + + public ProtocolStatus newInstance(){ + return newBuilder().build(); + } + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends ProtocolStatus implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'code' field. + */ + public Integer getCode() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'code' field. + * @param value the value to set. + */ + public void setCode(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isCodeDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'args' field. + */ + public java.util.List getArgs() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'args' field. + * @param value the value to set. + */ + public void setArgs(java.util.List value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isArgsDirty(java.util.List value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'lastModified' field. + */ + public Long getLastModified() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'lastModified' field. + * @param value the value to set. + */ + public void setLastModified(Long value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isLastModifiedDirty(Long value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + + } + +} \ No newline at end of file diff --git src/java/org/apache/nutch/storage/StorageUtils.java src/java/org/apache/nutch/storage/StorageUtils.java index 1540e3e..4bdc775 100644 --- src/java/org/apache/nutch/storage/StorageUtils.java +++ src/java/org/apache/nutch/storage/StorageUtils.java @@ -16,10 +16,6 @@ ******************************************************************************/ package org.apache.nutch.storage; -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; - import org.apache.gora.mapreduce.GoraMapper; import org.apache.gora.mapreduce.GoraOutputFormat; import org.apache.gora.mapreduce.GoraReducer; @@ -33,6 +29,10 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Partitioner; import org.apache.nutch.metadata.Nutch; +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; + /** * Entry point to Gora store/mapreduce functionality. * Translates the concept of "crawlid" to the corresponding Gora support. @@ -52,23 +52,23 @@ public class StorageUtils { @SuppressWarnings("unchecked") public static DataStore createWebStore(Configuration conf, Class keyClass, Class persistentClass) throws ClassNotFoundException, GoraException { - - String schema = null; + + String crawlId = conf.get(Nutch.CRAWL_ID_KEY, ""); + String schemaPrefix = ""; + if (!crawlId.isEmpty()) { + schemaPrefix = crawlId + "_"; + } + + String schema; if (WebPage.class.equals(persistentClass)) { schema = conf.get("storage.schema.webpage", "webpage"); + conf.set("preferred.schema.name", schemaPrefix + "webpage"); } else if (Host.class.equals(persistentClass)) { schema = conf.get("storage.schema.host", "host"); + conf.set("preferred.schema.name", schemaPrefix + "host"); } else { throw new UnsupportedOperationException("Unable to create store for class " + persistentClass); } - - String crawlId = conf.get(Nutch.CRAWL_ID_KEY, ""); - - if (!crawlId.isEmpty()) { - conf.set("schema.prefix", crawlId + "_"); - } else { - conf.set("schema.prefix", ""); - } Class> dataStoreClass = (Class>) getDataStoreClass(conf); diff --git src/java/org/apache/nutch/storage/WebPage.java src/java/org/apache/nutch/storage/WebPage.java index ddfc6b9..eaba3e2 100644 --- src/java/org/apache/nutch/storage/WebPage.java +++ src/java/org/apache/nutch/storage/WebPage.java @@ -1,84 +1,108 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at +/** + * Autogenerated by Avro * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.nutch.storage; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.HashMap; -import org.apache.avro.Protocol; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Protocol; -import org.apache.avro.util.Utf8; -import org.apache.avro.ipc.AvroRemoteException; -import org.apache.avro.generic.GenericArray; -import org.apache.avro.specific.FixedSize; -import org.apache.avro.specific.SpecificExceptionBase; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.avro.specific.SpecificRecord; -import org.apache.avro.specific.SpecificFixed; -import org.apache.gora.persistency.StateManager; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.persistency.impl.StateManagerImpl; -import org.apache.gora.persistency.StatefulHashMap; -import org.apache.gora.persistency.ListGenericArray; - + * DO NOT EDIT DIRECTLY + */ +package org.apache.nutch.storage; @SuppressWarnings("all") -public class WebPage extends PersistentBase { - public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"baseUrl\",\"type\":\"string\"},{\"name\":\"status\",\"type\":\"int\"},{\"name\":\"fetchTime\",\"type\":\"long\"},{\"name\":\"prevFetchTime\",\"type\":\"long\"},{\"name\":\"fetchInterval\",\"type\":\"int\"},{\"name\":\"retriesSinceFetch\",\"type\":\"int\"},{\"name\":\"modifiedTime\",\"type\":\"long\"},{\"name\":\"prevModifiedTime\",\"type\":\"long\"},{\"name\":\"protocolStatus\",\"type\":{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"fields\":[{\"name\":\"code\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"lastModified\",\"type\":\"long\"}]}},{\"name\":\"content\",\"type\":\"bytes\"},{\"name\":\"contentType\",\"type\":\"string\"},{\"name\":\"prevSignature\",\"type\":\"bytes\"},{\"name\":\"signature\",\"type\":\"bytes\"},{\"name\":\"title\",\"type\":\"string\"},{\"name\":\"text\",\"type\":\"string\"},{\"name\":\"parseStatus\",\"type\":{\"type\":\"record\",\"name\":\"ParseStatus\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\"},{\"name\":\"minorCode\",\"type\":\"int\"},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}},{\"name\":\"score\",\"type\":\"float\"},{\"name\":\"reprUrl\",\"type\":\"string\"},{\"name\":\"headers\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"markers\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"}},{\"name\":\"batchId\",\"type\":\"string\"}]}"); +public class WebPage extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AAAAAA==\"},{\"name\":\"baseUrl\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"status\",\"type\":\"int\",\"default\":0},{\"name\":\"fetchTime\",\"type\":\"long\",\"default\":0},{\"name\":\"prevFetchTime\",\"type\":\"long\",\"default\":0},{\"name\":\"fetchInterval\",\"type\":\"int\",\"default\":0},{\"name\":\"retriesSinceFetch\",\"type\":\"int\",\"default\":0},{\"name\":\"modifiedTime\",\"type\":\"long\",\"default\":0},{\"name\":\"prevModifiedTime\",\"type\":\"long\",\"default\":0},{\"name\":\"protocolStatus\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"code\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"default\":0}]}],\"default\":null},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"contentType\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"prevSignature\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"signature\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"title\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"text\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"parseStatus\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"ParseStatus\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"majorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}],\"default\":null},{\"name\":\"score\",\"type\":\"float\",\"default\":0},{\"name\":\"reprUrl\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"headers\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"markers\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"bytes\"]},\"default\":{}},{\"name\":\"batchId\",\"type\":[\"null\",\"string\"],\"default\":null}]}"); + + /** Enum containing all data bean's fields. */ public static enum Field { - BASE_URL(0,"baseUrl"), - STATUS(1,"status"), - FETCH_TIME(2,"fetchTime"), - PREV_FETCH_TIME(3,"prevFetchTime"), - FETCH_INTERVAL(4,"fetchInterval"), - RETRIES_SINCE_FETCH(5,"retriesSinceFetch"), - MODIFIED_TIME(6,"modifiedTime"), - PREV_MODIFIED_TIME(7,"prevModifiedTime"), - PROTOCOL_STATUS(8,"protocolStatus"), - CONTENT(9,"content"), - CONTENT_TYPE(10,"contentType"), - PREV_SIGNATURE(11,"prevSignature"), - SIGNATURE(12,"signature"), - TITLE(13,"title"), - TEXT(14,"text"), - PARSE_STATUS(15,"parseStatus"), - SCORE(16,"score"), - REPR_URL(17,"reprUrl"), - HEADERS(18,"headers"), - OUTLINKS(19,"outlinks"), - INLINKS(20,"inlinks"), - MARKERS(21,"markers"), - METADATA(22,"metadata"), - BATCH_ID(23,"batchId"), + __G__DIRTY(0, "__g__dirty"), + BASE_URL(1, "baseUrl"), + STATUS(2, "status"), + FETCH_TIME(3, "fetchTime"), + PREV_FETCH_TIME(4, "prevFetchTime"), + FETCH_INTERVAL(5, "fetchInterval"), + RETRIES_SINCE_FETCH(6, "retriesSinceFetch"), + MODIFIED_TIME(7, "modifiedTime"), + PREV_MODIFIED_TIME(8, "prevModifiedTime"), + PROTOCOL_STATUS(9, "protocolStatus"), + CONTENT(10, "content"), + CONTENT_TYPE(11, "contentType"), + PREV_SIGNATURE(12, "prevSignature"), + SIGNATURE(13, "signature"), + TITLE(14, "title"), + TEXT(15, "text"), + PARSE_STATUS(16, "parseStatus"), + SCORE(17, "score"), + REPR_URL(18, "reprUrl"), + HEADERS(19, "headers"), + OUTLINKS(20, "outlinks"), + INLINKS(21, "inlinks"), + MARKERS(22, "markers"), + METADATA(23, "metadata"), + BATCH_ID(24, "batchId"), ; + /** + * Field's index. + */ private int index; + + /** + * Field's name. + */ private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ public String toString() {return name;} }; - public static final String[] _ALL_FIELDS = {"baseUrl","status","fetchTime","prevFetchTime","fetchInterval","retriesSinceFetch","modifiedTime","prevModifiedTime","protocolStatus","content","contentType","prevSignature","signature","title","text","parseStatus","score","reprUrl","headers","outlinks","inlinks","markers","metadata","batchId",}; - static { - PersistentBase.registerFields(WebPage.class, _ALL_FIELDS); - } - private Utf8 baseUrl; + + public static final String[] _ALL_FIELDS = { + "__g__dirty", + "baseUrl", + "status", + "fetchTime", + "prevFetchTime", + "fetchInterval", + "retriesSinceFetch", + "modifiedTime", + "prevModifiedTime", + "protocolStatus", + "content", + "contentType", + "prevSignature", + "signature", + "title", + "text", + "parseStatus", + "score", + "reprUrl", + "headers", + "outlinks", + "inlinks", + "markers", + "metadata", + "batchId", + }; + + /** Bytes used to represent weather or not a field is dirty. */ + private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[4]); + private CharSequence baseUrl; private int status; private long fetchTime; private long prevFetchTime; @@ -87,294 +111,2041 @@ public class WebPage extends PersistentBase { private long modifiedTime; private long prevModifiedTime; private ProtocolStatus protocolStatus; - private ByteBuffer content; - private Utf8 contentType; - private ByteBuffer prevSignature; - private ByteBuffer signature; - private Utf8 title; - private Utf8 text; + private java.nio.ByteBuffer content; + private CharSequence contentType; + private java.nio.ByteBuffer prevSignature; + private java.nio.ByteBuffer signature; + private CharSequence title; + private CharSequence text; private ParseStatus parseStatus; private float score; - private Utf8 reprUrl; - private Map headers; - private Map outlinks; - private Map inlinks; - private Map markers; - private Map metadata; - private Utf8 batchId; - public WebPage() { - this(new StateManagerImpl()); - } - public WebPage(StateManager stateManager) { - super(stateManager); - headers = new StatefulHashMap(); - outlinks = new StatefulHashMap(); - inlinks = new StatefulHashMap(); - markers = new StatefulHashMap(); - metadata = new StatefulHashMap(); - } - public WebPage newInstance(StateManager stateManager) { - return new WebPage(stateManager); - } - public Schema getSchema() { return _SCHEMA; } - public Object get(int _field) { - switch (_field) { - case 0: return baseUrl; - case 1: return status; - case 2: return fetchTime; - case 3: return prevFetchTime; - case 4: return fetchInterval; - case 5: return retriesSinceFetch; - case 6: return modifiedTime; - case 7: return prevModifiedTime; - case 8: return protocolStatus; - case 9: return content; - case 10: return contentType; - case 11: return prevSignature; - case 12: return signature; - case 13: return title; - case 14: return text; - case 15: return parseStatus; - case 16: return score; - case 17: return reprUrl; - case 18: return headers; - case 19: return outlinks; - case 20: return inlinks; - case 21: return markers; - case 22: return metadata; - case 23: return batchId; - default: throw new AvroRuntimeException("Bad index"); + private CharSequence reprUrl; + private java.util.Map headers; + private java.util.Map outlinks; + private java.util.Map inlinks; + private java.util.Map markers; + private java.util.Map metadata; + private CharSequence batchId; + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public Object get(int field$) { + switch (field$) { + case 0: return __g__dirty; + case 1: return baseUrl; + case 2: return status; + case 3: return fetchTime; + case 4: return prevFetchTime; + case 5: return fetchInterval; + case 6: return retriesSinceFetch; + case 7: return modifiedTime; + case 8: return prevModifiedTime; + case 9: return protocolStatus; + case 10: return content; + case 11: return contentType; + case 12: return prevSignature; + case 13: return signature; + case 14: return title; + case 15: return text; + case 16: return parseStatus; + case 17: return score; + case 18: return reprUrl; + case 19: return headers; + case 20: return outlinks; + case 21: return inlinks; + case 22: return markers; + case 23: return metadata; + case 24: return batchId; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } + + // Used by DatumReader. Applications should not call. @SuppressWarnings(value="unchecked") - public void put(int _field, Object _value) { - if(isFieldEqual(_field, _value)) return; - getStateManager().setDirty(this, _field); - switch (_field) { - case 0:baseUrl = (Utf8)_value; break; - case 1:status = (Integer)_value; break; - case 2:fetchTime = (Long)_value; break; - case 3:prevFetchTime = (Long)_value; break; - case 4:fetchInterval = (Integer)_value; break; - case 5:retriesSinceFetch = (Integer)_value; break; - case 6:modifiedTime = (Long)_value; break; - case 7:prevModifiedTime = (Long)_value; break; - case 8:protocolStatus = (ProtocolStatus)_value; break; - case 9:content = (ByteBuffer)_value; break; - case 10:contentType = (Utf8)_value; break; - case 11:prevSignature = (ByteBuffer)_value; break; - case 12:signature = (ByteBuffer)_value; break; - case 13:title = (Utf8)_value; break; - case 14:text = (Utf8)_value; break; - case 15:parseStatus = (ParseStatus)_value; break; - case 16:score = (Float)_value; break; - case 17:reprUrl = (Utf8)_value; break; - case 18:headers = (Map)_value; break; - case 19:outlinks = (Map)_value; break; - case 20:inlinks = (Map)_value; break; - case 21:markers = (Map)_value; break; - case 22:metadata = (Map)_value; break; - case 23:batchId = (Utf8)_value; break; - default: throw new AvroRuntimeException("Bad index"); - } - } - public Utf8 getBaseUrl() { - return (Utf8) get(0); - } - public void setBaseUrl(Utf8 value) { - put(0, value); - } - public int getStatus() { - return (Integer) get(1); - } - public void setStatus(int value) { - put(1, value); - } - public long getFetchTime() { - return (Long) get(2); - } - public void setFetchTime(long value) { - put(2, value); - } - public long getPrevFetchTime() { - return (Long) get(3); - } - public void setPrevFetchTime(long value) { - put(3, value); - } - public int getFetchInterval() { - return (Integer) get(4); - } - public void setFetchInterval(int value) { - put(4, value); - } - public int getRetriesSinceFetch() { - return (Integer) get(5); - } - public void setRetriesSinceFetch(int value) { - put(5, value); - } - public long getModifiedTime() { - return (Long) get(6); - } - public void setModifiedTime(long value) { - put(6, value); - } - public long getPrevModifiedTime() { - return (Long) get(7); - } - public void setPrevModifiedTime(long value) { - put(7, value); + public void put(int field$, Object value) { + switch (field$) { + case 0: __g__dirty = (java.nio.ByteBuffer)(value); break; + case 1: baseUrl = (CharSequence)(value); break; + case 2: status = (Integer)(value); break; + case 3: fetchTime = (Long)(value); break; + case 4: prevFetchTime = (Long)(value); break; + case 5: fetchInterval = (Integer)(value); break; + case 6: retriesSinceFetch = (Integer)(value); break; + case 7: modifiedTime = (Long)(value); break; + case 8: prevModifiedTime = (Long)(value); break; + case 9: protocolStatus = (ProtocolStatus)(value); break; + case 10: content = (java.nio.ByteBuffer)(value); break; + case 11: contentType = (CharSequence)(value); break; + case 12: prevSignature = (java.nio.ByteBuffer)(value); break; + case 13: signature = (java.nio.ByteBuffer)(value); break; + case 14: title = (CharSequence)(value); break; + case 15: text = (CharSequence)(value); break; + case 16: parseStatus = (ParseStatus)(value); break; + case 17: score = (Float)(value); break; + case 18: reprUrl = (CharSequence)(value); break; + case 19: headers = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 20: outlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 21: inlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 22: markers = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 23: metadata = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break; + case 24: batchId = (CharSequence)(value); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } } - public ProtocolStatus getProtocolStatus() { - return (ProtocolStatus) get(8); + + /** + * Gets the value of the 'baseUrl' field. + */ + public CharSequence getBaseUrl() { + return baseUrl; } - public void setProtocolStatus(ProtocolStatus value) { - put(8, value); + + /** + * Sets the value of the 'baseUrl' field. + * @param value the value to set. + */ + public void setBaseUrl(CharSequence value) { + this.baseUrl = value; + setDirty(1); + } + + /** + * Checks the dirty status of the 'baseUrl' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isBaseUrlDirty(CharSequence value) { + return isDirty(1); } - public ByteBuffer getContent() { - return (ByteBuffer) get(9); + + /** + * Gets the value of the 'status' field. + */ + public Integer getStatus() { + return status; } - public void setContent(ByteBuffer value) { - put(9, value); + + /** + * Sets the value of the 'status' field. + * @param value the value to set. + */ + public void setStatus(Integer value) { + this.status = value; + setDirty(2); + } + + /** + * Checks the dirty status of the 'status' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isStatusDirty(Integer value) { + return isDirty(2); } - public Utf8 getContentType() { - return (Utf8) get(10); + + /** + * Gets the value of the 'fetchTime' field. + */ + public Long getFetchTime() { + return fetchTime; } - public void setContentType(Utf8 value) { - put(10, value); + + /** + * Sets the value of the 'fetchTime' field. + * @param value the value to set. + */ + public void setFetchTime(Long value) { + this.fetchTime = value; + setDirty(3); + } + + /** + * Checks the dirty status of the 'fetchTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isFetchTimeDirty(Long value) { + return isDirty(3); } - public ByteBuffer getPrevSignature() { - return (ByteBuffer) get(11); + + /** + * Gets the value of the 'prevFetchTime' field. + */ + public Long getPrevFetchTime() { + return prevFetchTime; } - public void setPrevSignature(ByteBuffer value) { - put(11, value); + + /** + * Sets the value of the 'prevFetchTime' field. + * @param value the value to set. + */ + public void setPrevFetchTime(Long value) { + this.prevFetchTime = value; + setDirty(4); + } + + /** + * Checks the dirty status of the 'prevFetchTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isPrevFetchTimeDirty(Long value) { + return isDirty(4); } - public ByteBuffer getSignature() { - return (ByteBuffer) get(12); + + /** + * Gets the value of the 'fetchInterval' field. + */ + public Integer getFetchInterval() { + return fetchInterval; } - public void setSignature(ByteBuffer value) { - put(12, value); + + /** + * Sets the value of the 'fetchInterval' field. + * @param value the value to set. + */ + public void setFetchInterval(Integer value) { + this.fetchInterval = value; + setDirty(5); + } + + /** + * Checks the dirty status of the 'fetchInterval' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isFetchIntervalDirty(Integer value) { + return isDirty(5); } - public Utf8 getTitle() { - return (Utf8) get(13); + + /** + * Gets the value of the 'retriesSinceFetch' field. + */ + public Integer getRetriesSinceFetch() { + return retriesSinceFetch; } - public void setTitle(Utf8 value) { - put(13, value); + + /** + * Sets the value of the 'retriesSinceFetch' field. + * @param value the value to set. + */ + public void setRetriesSinceFetch(Integer value) { + this.retriesSinceFetch = value; + setDirty(6); + } + + /** + * Checks the dirty status of the 'retriesSinceFetch' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isRetriesSinceFetchDirty(Integer value) { + return isDirty(6); } - public Utf8 getText() { - return (Utf8) get(14); + + /** + * Gets the value of the 'modifiedTime' field. + */ + public Long getModifiedTime() { + return modifiedTime; } - public void setText(Utf8 value) { - put(14, value); + + /** + * Sets the value of the 'modifiedTime' field. + * @param value the value to set. + */ + public void setModifiedTime(Long value) { + this.modifiedTime = value; + setDirty(7); + } + + /** + * Checks the dirty status of the 'modifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isModifiedTimeDirty(Long value) { + return isDirty(7); } - public ParseStatus getParseStatus() { - return (ParseStatus) get(15); + + /** + * Gets the value of the 'prevModifiedTime' field. + */ + public Long getPrevModifiedTime() { + return prevModifiedTime; } - public void setParseStatus(ParseStatus value) { - put(15, value); + + /** + * Sets the value of the 'prevModifiedTime' field. + * @param value the value to set. + */ + public void setPrevModifiedTime(Long value) { + this.prevModifiedTime = value; + setDirty(8); + } + + /** + * Checks the dirty status of the 'prevModifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isPrevModifiedTimeDirty(Long value) { + return isDirty(8); } - public float getScore() { - return (Float) get(16); + + /** + * Gets the value of the 'protocolStatus' field. + */ + public ProtocolStatus getProtocolStatus() { + return protocolStatus; } - public void setScore(float value) { - put(16, value); + + /** + * Sets the value of the 'protocolStatus' field. + * @param value the value to set. + */ + public void setProtocolStatus(ProtocolStatus value) { + this.protocolStatus = value; + setDirty(9); } - public Utf8 getReprUrl() { - return (Utf8) get(17); + + /** + * Checks the dirty status of the 'protocolStatus' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isProtocolStatusDirty(ProtocolStatus value) { + return isDirty(9); } - public void setReprUrl(Utf8 value) { - put(17, value); + + /** + * Gets the value of the 'content' field. + */ + public java.nio.ByteBuffer getContent() { + return content; } - @SuppressWarnings("unchecked") - public Map getHeaders() { - return (Map) get(18); + + /** + * Sets the value of the 'content' field. + * @param value the value to set. + */ + public void setContent(java.nio.ByteBuffer value) { + this.content = value; + setDirty(10); + } + + /** + * Checks the dirty status of the 'content' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isContentDirty(java.nio.ByteBuffer value) { + return isDirty(10); } - public Utf8 getFromHeaders(Utf8 key) { - if (headers == null) { return null; } - return headers.get(key); + + /** + * Gets the value of the 'contentType' field. + */ + public CharSequence getContentType() { + return contentType; } - public void putToHeaders(Utf8 key, Utf8 value) { - getStateManager().setDirty(this, 18); - headers.put(key, value); + + /** + * Sets the value of the 'contentType' field. + * @param value the value to set. + */ + public void setContentType(CharSequence value) { + this.contentType = value; + setDirty(11); + } + + /** + * Checks the dirty status of the 'contentType' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isContentTypeDirty(CharSequence value) { + return isDirty(11); } - public Utf8 removeFromHeaders(Utf8 key) { - if (headers == null) { return null; } - getStateManager().setDirty(this, 18); - return headers.remove(key); + + /** + * Gets the value of the 'prevSignature' field. + */ + public java.nio.ByteBuffer getPrevSignature() { + return prevSignature; } - @SuppressWarnings("unchecked") - public Map getOutlinks() { - return (Map) get(19); + + /** + * Sets the value of the 'prevSignature' field. + * @param value the value to set. + */ + public void setPrevSignature(java.nio.ByteBuffer value) { + this.prevSignature = value; + setDirty(12); + } + + /** + * Checks the dirty status of the 'prevSignature' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isPrevSignatureDirty(java.nio.ByteBuffer value) { + return isDirty(12); } - public Utf8 getFromOutlinks(Utf8 key) { - if (outlinks == null) { return null; } - return outlinks.get(key); + + /** + * Gets the value of the 'signature' field. + */ + public java.nio.ByteBuffer getSignature() { + return signature; } - public void putToOutlinks(Utf8 key, Utf8 value) { - getStateManager().setDirty(this, 19); - outlinks.put(key, value); + + /** + * Sets the value of the 'signature' field. + * @param value the value to set. + */ + public void setSignature(java.nio.ByteBuffer value) { + this.signature = value; + setDirty(13); + } + + /** + * Checks the dirty status of the 'signature' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isSignatureDirty(java.nio.ByteBuffer value) { + return isDirty(13); } - public Utf8 removeFromOutlinks(Utf8 key) { - if (outlinks == null) { return null; } - getStateManager().setDirty(this, 19); - return outlinks.remove(key); + + /** + * Gets the value of the 'title' field. + */ + public CharSequence getTitle() { + return title; } - @SuppressWarnings("unchecked") - public Map getInlinks() { - return (Map) get(20); + + /** + * Sets the value of the 'title' field. + * @param value the value to set. + */ + public void setTitle(CharSequence value) { + this.title = value; + setDirty(14); + } + + /** + * Checks the dirty status of the 'title' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isTitleDirty(CharSequence value) { + return isDirty(14); } - public Utf8 getFromInlinks(Utf8 key) { - if (inlinks == null) { return null; } - return inlinks.get(key); + + /** + * Gets the value of the 'text' field. + */ + public CharSequence getText() { + return text; + } + + /** + * Sets the value of the 'text' field. + * @param value the value to set. + */ + public void setText(CharSequence value) { + this.text = value; + setDirty(15); + } + + /** + * Checks the dirty status of the 'text' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isTextDirty(CharSequence value) { + return isDirty(15); + } + + /** + * Gets the value of the 'parseStatus' field. + */ + public ParseStatus getParseStatus() { + return parseStatus; + } + + /** + * Sets the value of the 'parseStatus' field. + * @param value the value to set. + */ + public void setParseStatus(ParseStatus value) { + this.parseStatus = value; + setDirty(16); + } + + /** + * Checks the dirty status of the 'parseStatus' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isParseStatusDirty(ParseStatus value) { + return isDirty(16); + } + + /** + * Gets the value of the 'score' field. + */ + public Float getScore() { + return score; + } + + /** + * Sets the value of the 'score' field. + * @param value the value to set. + */ + public void setScore(Float value) { + this.score = value; + setDirty(17); + } + + /** + * Checks the dirty status of the 'score' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isScoreDirty(Float value) { + return isDirty(17); + } + + /** + * Gets the value of the 'reprUrl' field. + */ + public CharSequence getReprUrl() { + return reprUrl; + } + + /** + * Sets the value of the 'reprUrl' field. + * @param value the value to set. + */ + public void setReprUrl(CharSequence value) { + this.reprUrl = value; + setDirty(18); + } + + /** + * Checks the dirty status of the 'reprUrl' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isReprUrlDirty(CharSequence value) { + return isDirty(18); + } + + /** + * Gets the value of the 'headers' field. + */ + public java.util.Map getHeaders() { + return headers; + } + + /** + * Sets the value of the 'headers' field. + * @param value the value to set. + */ + public void setHeaders(java.util.Map value) { + this.headers = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(19); + } + + /** + * Checks the dirty status of the 'headers' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isHeadersDirty(java.util.Map value) { + return isDirty(19); + } + + /** + * Gets the value of the 'outlinks' field. + */ + public java.util.Map getOutlinks() { + return outlinks; } - public void putToInlinks(Utf8 key, Utf8 value) { - getStateManager().setDirty(this, 20); - inlinks.put(key, value); + + /** + * Sets the value of the 'outlinks' field. + * @param value the value to set. + */ + public void setOutlinks(java.util.Map value) { + this.outlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(20); + } + + /** + * Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isOutlinksDirty(java.util.Map value) { + return isDirty(20); } - public Utf8 removeFromInlinks(Utf8 key) { - if (inlinks == null) { return null; } - getStateManager().setDirty(this, 20); - return inlinks.remove(key); + + /** + * Gets the value of the 'inlinks' field. + */ + public java.util.Map getInlinks() { + return inlinks; } - @SuppressWarnings("unchecked") - public Map getMarkers() { - return (Map) get(21); + + /** + * Sets the value of the 'inlinks' field. + * @param value the value to set. + */ + public void setInlinks(java.util.Map value) { + this.inlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(21); + } + + /** + * Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isInlinksDirty(java.util.Map value) { + return isDirty(21); } - public Utf8 getFromMarkers(Utf8 key) { - if (markers == null) { return null; } - return markers.get(key); + + /** + * Gets the value of the 'markers' field. + */ + public java.util.Map getMarkers() { + return markers; } - public void putToMarkers(Utf8 key, Utf8 value) { - getStateManager().setDirty(this, 21); - markers.put(key, value); + + /** + * Sets the value of the 'markers' field. + * @param value the value to set. + */ + public void setMarkers(java.util.Map value) { + this.markers = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(22); + } + + /** + * Checks the dirty status of the 'markers' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMarkersDirty(java.util.Map value) { + return isDirty(22); } - public Utf8 removeFromMarkers(Utf8 key) { - if (markers == null) { return null; } - getStateManager().setDirty(this, 21); - return markers.remove(key); + + /** + * Gets the value of the 'metadata' field. + */ + public java.util.Map getMetadata() { + return metadata; } - @SuppressWarnings("unchecked") - public Map getMetadata() { - return (Map) get(22); + + /** + * Sets the value of the 'metadata' field. + * @param value the value to set. + */ + public void setMetadata(java.util.Map value) { + this.metadata = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value); + setDirty(23); + } + + /** + * Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMetadataDirty(java.util.Map value) { + return isDirty(23); } - public ByteBuffer getFromMetadata(Utf8 key) { - if (metadata == null) { return null; } - return metadata.get(key); + + /** + * Gets the value of the 'batchId' field. + */ + public CharSequence getBatchId() { + return batchId; } - public void putToMetadata(Utf8 key, ByteBuffer value) { - getStateManager().setDirty(this, 22); - metadata.put(key, value); + + /** + * Sets the value of the 'batchId' field. + * @param value the value to set. + */ + public void setBatchId(CharSequence value) { + this.batchId = value; + setDirty(24); + } + + /** + * Checks the dirty status of the 'batchId' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isBatchIdDirty(CharSequence value) { + return isDirty(24); } - public ByteBuffer removeFromMetadata(Utf8 key) { - if (metadata == null) { return null; } - getStateManager().setDirty(this, 22); - return metadata.remove(key); + + /** Creates a new WebPage RecordBuilder */ + public static Builder newBuilder() { + return new Builder(); + } + + /** Creates a new WebPage RecordBuilder by copying an existing Builder */ + public static Builder newBuilder(Builder other) { + return new Builder(other); + } + + /** Creates a new WebPage RecordBuilder by copying an existing WebPage instance */ + public static Builder newBuilder(WebPage other) { + return new Builder(other); + } + + private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); + } + + /** + * RecordBuilder for WebPage instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.nio.ByteBuffer __g__dirty; + private CharSequence baseUrl; + private int status; + private long fetchTime; + private long prevFetchTime; + private int fetchInterval; + private int retriesSinceFetch; + private long modifiedTime; + private long prevModifiedTime; + private ProtocolStatus protocolStatus; + private java.nio.ByteBuffer content; + private CharSequence contentType; + private java.nio.ByteBuffer prevSignature; + private java.nio.ByteBuffer signature; + private CharSequence title; + private CharSequence text; + private ParseStatus parseStatus; + private float score; + private CharSequence reprUrl; + private java.util.Map headers; + private java.util.Map outlinks; + private java.util.Map inlinks; + private java.util.Map markers; + private java.util.Map metadata; + private CharSequence batchId; + + /** Creates a new Builder */ + private Builder() { + super(WebPage.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing WebPage instance */ + private Builder(WebPage other) { + super(WebPage.SCHEMA$); + if (isValidValue(fields()[0], other.__g__dirty)) { + this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.baseUrl)) { + this.baseUrl = (CharSequence) data().deepCopy(fields()[1].schema(), other.baseUrl); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.status)) { + this.status = (Integer) data().deepCopy(fields()[2].schema(), other.status); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.fetchTime)) { + this.fetchTime = (Long) data().deepCopy(fields()[3].schema(), other.fetchTime); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.prevFetchTime)) { + this.prevFetchTime = (Long) data().deepCopy(fields()[4].schema(), other.prevFetchTime); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.fetchInterval)) { + this.fetchInterval = (Integer) data().deepCopy(fields()[5].schema(), other.fetchInterval); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.retriesSinceFetch)) { + this.retriesSinceFetch = (Integer) data().deepCopy(fields()[6].schema(), other.retriesSinceFetch); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.modifiedTime)) { + this.modifiedTime = (Long) data().deepCopy(fields()[7].schema(), other.modifiedTime); + fieldSetFlags()[7] = true; + } + if (isValidValue(fields()[8], other.prevModifiedTime)) { + this.prevModifiedTime = (Long) data().deepCopy(fields()[8].schema(), other.prevModifiedTime); + fieldSetFlags()[8] = true; + } + if (isValidValue(fields()[9], other.protocolStatus)) { + this.protocolStatus = (ProtocolStatus) data().deepCopy(fields()[9].schema(), other.protocolStatus); + fieldSetFlags()[9] = true; + } + if (isValidValue(fields()[10], other.content)) { + this.content = (java.nio.ByteBuffer) data().deepCopy(fields()[10].schema(), other.content); + fieldSetFlags()[10] = true; + } + if (isValidValue(fields()[11], other.contentType)) { + this.contentType = (CharSequence) data().deepCopy(fields()[11].schema(), other.contentType); + fieldSetFlags()[11] = true; + } + if (isValidValue(fields()[12], other.prevSignature)) { + this.prevSignature = (java.nio.ByteBuffer) data().deepCopy(fields()[12].schema(), other.prevSignature); + fieldSetFlags()[12] = true; + } + if (isValidValue(fields()[13], other.signature)) { + this.signature = (java.nio.ByteBuffer) data().deepCopy(fields()[13].schema(), other.signature); + fieldSetFlags()[13] = true; + } + if (isValidValue(fields()[14], other.title)) { + this.title = (CharSequence) data().deepCopy(fields()[14].schema(), other.title); + fieldSetFlags()[14] = true; + } + if (isValidValue(fields()[15], other.text)) { + this.text = (CharSequence) data().deepCopy(fields()[15].schema(), other.text); + fieldSetFlags()[15] = true; + } + if (isValidValue(fields()[16], other.parseStatus)) { + this.parseStatus = (ParseStatus) data().deepCopy(fields()[16].schema(), other.parseStatus); + fieldSetFlags()[16] = true; + } + if (isValidValue(fields()[17], other.score)) { + this.score = (Float) data().deepCopy(fields()[17].schema(), other.score); + fieldSetFlags()[17] = true; + } + if (isValidValue(fields()[18], other.reprUrl)) { + this.reprUrl = (CharSequence) data().deepCopy(fields()[18].schema(), other.reprUrl); + fieldSetFlags()[18] = true; + } + if (isValidValue(fields()[19], other.headers)) { + this.headers = (java.util.Map) data().deepCopy(fields()[19].schema(), other.headers); + fieldSetFlags()[19] = true; + } + if (isValidValue(fields()[20], other.outlinks)) { + this.outlinks = (java.util.Map) data().deepCopy(fields()[20].schema(), other.outlinks); + fieldSetFlags()[20] = true; + } + if (isValidValue(fields()[21], other.inlinks)) { + this.inlinks = (java.util.Map) data().deepCopy(fields()[21].schema(), other.inlinks); + fieldSetFlags()[21] = true; + } + if (isValidValue(fields()[22], other.markers)) { + this.markers = (java.util.Map) data().deepCopy(fields()[22].schema(), other.markers); + fieldSetFlags()[22] = true; + } + if (isValidValue(fields()[23], other.metadata)) { + this.metadata = (java.util.Map) data().deepCopy(fields()[23].schema(), other.metadata); + fieldSetFlags()[23] = true; + } + if (isValidValue(fields()[24], other.batchId)) { + this.batchId = (CharSequence) data().deepCopy(fields()[24].schema(), other.batchId); + fieldSetFlags()[24] = true; + } + } + + /** Gets the value of the 'baseUrl' field */ + public CharSequence getBaseUrl() { + return baseUrl; + } + + /** Sets the value of the 'baseUrl' field */ + public Builder setBaseUrl(CharSequence value) { + validate(fields()[1], value); + this.baseUrl = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'baseUrl' field has been set */ + public boolean hasBaseUrl() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'baseUrl' field */ + public Builder clearBaseUrl() { + baseUrl = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'status' field */ + public Integer getStatus() { + return status; + } + + /** Sets the value of the 'status' field */ + public Builder setStatus(int value) { + validate(fields()[2], value); + this.status = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'status' field has been set */ + public boolean hasStatus() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'status' field */ + public Builder clearStatus() { + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'fetchTime' field */ + public Long getFetchTime() { + return fetchTime; + } + + /** Sets the value of the 'fetchTime' field */ + public Builder setFetchTime(long value) { + validate(fields()[3], value); + this.fetchTime = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'fetchTime' field has been set */ + public boolean hasFetchTime() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'fetchTime' field */ + public Builder clearFetchTime() { + fieldSetFlags()[3] = false; + return this; + } + + /** Gets the value of the 'prevFetchTime' field */ + public Long getPrevFetchTime() { + return prevFetchTime; + } + + /** Sets the value of the 'prevFetchTime' field */ + public Builder setPrevFetchTime(long value) { + validate(fields()[4], value); + this.prevFetchTime = value; + fieldSetFlags()[4] = true; + return this; + } + + /** Checks whether the 'prevFetchTime' field has been set */ + public boolean hasPrevFetchTime() { + return fieldSetFlags()[4]; + } + + /** Clears the value of the 'prevFetchTime' field */ + public Builder clearPrevFetchTime() { + fieldSetFlags()[4] = false; + return this; + } + + /** Gets the value of the 'fetchInterval' field */ + public Integer getFetchInterval() { + return fetchInterval; + } + + /** Sets the value of the 'fetchInterval' field */ + public Builder setFetchInterval(int value) { + validate(fields()[5], value); + this.fetchInterval = value; + fieldSetFlags()[5] = true; + return this; + } + + /** Checks whether the 'fetchInterval' field has been set */ + public boolean hasFetchInterval() { + return fieldSetFlags()[5]; + } + + /** Clears the value of the 'fetchInterval' field */ + public Builder clearFetchInterval() { + fieldSetFlags()[5] = false; + return this; + } + + /** Gets the value of the 'retriesSinceFetch' field */ + public Integer getRetriesSinceFetch() { + return retriesSinceFetch; + } + + /** Sets the value of the 'retriesSinceFetch' field */ + public Builder setRetriesSinceFetch(int value) { + validate(fields()[6], value); + this.retriesSinceFetch = value; + fieldSetFlags()[6] = true; + return this; + } + + /** Checks whether the 'retriesSinceFetch' field has been set */ + public boolean hasRetriesSinceFetch() { + return fieldSetFlags()[6]; + } + + /** Clears the value of the 'retriesSinceFetch' field */ + public Builder clearRetriesSinceFetch() { + fieldSetFlags()[6] = false; + return this; + } + + /** Gets the value of the 'modifiedTime' field */ + public Long getModifiedTime() { + return modifiedTime; + } + + /** Sets the value of the 'modifiedTime' field */ + public Builder setModifiedTime(long value) { + validate(fields()[7], value); + this.modifiedTime = value; + fieldSetFlags()[7] = true; + return this; + } + + /** Checks whether the 'modifiedTime' field has been set */ + public boolean hasModifiedTime() { + return fieldSetFlags()[7]; + } + + /** Clears the value of the 'modifiedTime' field */ + public Builder clearModifiedTime() { + fieldSetFlags()[7] = false; + return this; + } + + /** Gets the value of the 'prevModifiedTime' field */ + public Long getPrevModifiedTime() { + return prevModifiedTime; + } + + /** Sets the value of the 'prevModifiedTime' field */ + public Builder setPrevModifiedTime(long value) { + validate(fields()[8], value); + this.prevModifiedTime = value; + fieldSetFlags()[8] = true; + return this; + } + + /** Checks whether the 'prevModifiedTime' field has been set */ + public boolean hasPrevModifiedTime() { + return fieldSetFlags()[8]; + } + + /** Clears the value of the 'prevModifiedTime' field */ + public Builder clearPrevModifiedTime() { + fieldSetFlags()[8] = false; + return this; + } + + /** Gets the value of the 'protocolStatus' field */ + public ProtocolStatus getProtocolStatus() { + return protocolStatus; + } + + /** Sets the value of the 'protocolStatus' field */ + public Builder setProtocolStatus(ProtocolStatus value) { + validate(fields()[9], value); + this.protocolStatus = value; + fieldSetFlags()[9] = true; + return this; + } + + /** Checks whether the 'protocolStatus' field has been set */ + public boolean hasProtocolStatus() { + return fieldSetFlags()[9]; + } + + /** Clears the value of the 'protocolStatus' field */ + public Builder clearProtocolStatus() { + protocolStatus = null; + fieldSetFlags()[9] = false; + return this; + } + + /** Gets the value of the 'content' field */ + public java.nio.ByteBuffer getContent() { + return content; + } + + /** Sets the value of the 'content' field */ + public Builder setContent(java.nio.ByteBuffer value) { + validate(fields()[10], value); + this.content = value; + fieldSetFlags()[10] = true; + return this; + } + + /** Checks whether the 'content' field has been set */ + public boolean hasContent() { + return fieldSetFlags()[10]; + } + + /** Clears the value of the 'content' field */ + public Builder clearContent() { + content = null; + fieldSetFlags()[10] = false; + return this; + } + + /** Gets the value of the 'contentType' field */ + public CharSequence getContentType() { + return contentType; + } + + /** Sets the value of the 'contentType' field */ + public Builder setContentType(CharSequence value) { + validate(fields()[11], value); + this.contentType = value; + fieldSetFlags()[11] = true; + return this; + } + + /** Checks whether the 'contentType' field has been set */ + public boolean hasContentType() { + return fieldSetFlags()[11]; + } + + /** Clears the value of the 'contentType' field */ + public Builder clearContentType() { + contentType = null; + fieldSetFlags()[11] = false; + return this; + } + + /** Gets the value of the 'prevSignature' field */ + public java.nio.ByteBuffer getPrevSignature() { + return prevSignature; + } + + /** Sets the value of the 'prevSignature' field */ + public Builder setPrevSignature(java.nio.ByteBuffer value) { + validate(fields()[12], value); + this.prevSignature = value; + fieldSetFlags()[12] = true; + return this; + } + + /** Checks whether the 'prevSignature' field has been set */ + public boolean hasPrevSignature() { + return fieldSetFlags()[12]; + } + + /** Clears the value of the 'prevSignature' field */ + public Builder clearPrevSignature() { + prevSignature = null; + fieldSetFlags()[12] = false; + return this; + } + + /** Gets the value of the 'signature' field */ + public java.nio.ByteBuffer getSignature() { + return signature; + } + + /** Sets the value of the 'signature' field */ + public Builder setSignature(java.nio.ByteBuffer value) { + validate(fields()[13], value); + this.signature = value; + fieldSetFlags()[13] = true; + return this; + } + + /** Checks whether the 'signature' field has been set */ + public boolean hasSignature() { + return fieldSetFlags()[13]; + } + + /** Clears the value of the 'signature' field */ + public Builder clearSignature() { + signature = null; + fieldSetFlags()[13] = false; + return this; + } + + /** Gets the value of the 'title' field */ + public CharSequence getTitle() { + return title; + } + + /** Sets the value of the 'title' field */ + public Builder setTitle(CharSequence value) { + validate(fields()[14], value); + this.title = value; + fieldSetFlags()[14] = true; + return this; + } + + /** Checks whether the 'title' field has been set */ + public boolean hasTitle() { + return fieldSetFlags()[14]; + } + + /** Clears the value of the 'title' field */ + public Builder clearTitle() { + title = null; + fieldSetFlags()[14] = false; + return this; + } + + /** Gets the value of the 'text' field */ + public CharSequence getText() { + return text; + } + + /** Sets the value of the 'text' field */ + public Builder setText(CharSequence value) { + validate(fields()[15], value); + this.text = value; + fieldSetFlags()[15] = true; + return this; + } + + /** Checks whether the 'text' field has been set */ + public boolean hasText() { + return fieldSetFlags()[15]; + } + + /** Clears the value of the 'text' field */ + public Builder clearText() { + text = null; + fieldSetFlags()[15] = false; + return this; + } + + /** Gets the value of the 'parseStatus' field */ + public ParseStatus getParseStatus() { + return parseStatus; + } + + /** Sets the value of the 'parseStatus' field */ + public Builder setParseStatus(ParseStatus value) { + validate(fields()[16], value); + this.parseStatus = value; + fieldSetFlags()[16] = true; + return this; + } + + /** Checks whether the 'parseStatus' field has been set */ + public boolean hasParseStatus() { + return fieldSetFlags()[16]; + } + + /** Clears the value of the 'parseStatus' field */ + public Builder clearParseStatus() { + parseStatus = null; + fieldSetFlags()[16] = false; + return this; + } + + /** Gets the value of the 'score' field */ + public Float getScore() { + return score; + } + + /** Sets the value of the 'score' field */ + public Builder setScore(float value) { + validate(fields()[17], value); + this.score = value; + fieldSetFlags()[17] = true; + return this; + } + + /** Checks whether the 'score' field has been set */ + public boolean hasScore() { + return fieldSetFlags()[17]; + } + + /** Clears the value of the 'score' field */ + public Builder clearScore() { + fieldSetFlags()[17] = false; + return this; + } + + /** Gets the value of the 'reprUrl' field */ + public CharSequence getReprUrl() { + return reprUrl; + } + + /** Sets the value of the 'reprUrl' field */ + public Builder setReprUrl(CharSequence value) { + validate(fields()[18], value); + this.reprUrl = value; + fieldSetFlags()[18] = true; + return this; + } + + /** Checks whether the 'reprUrl' field has been set */ + public boolean hasReprUrl() { + return fieldSetFlags()[18]; + } + + /** Clears the value of the 'reprUrl' field */ + public Builder clearReprUrl() { + reprUrl = null; + fieldSetFlags()[18] = false; + return this; + } + + /** Gets the value of the 'headers' field */ + public java.util.Map getHeaders() { + return headers; + } + + /** Sets the value of the 'headers' field */ + public Builder setHeaders(java.util.Map value) { + validate(fields()[19], value); + this.headers = value; + fieldSetFlags()[19] = true; + return this; + } + + /** Checks whether the 'headers' field has been set */ + public boolean hasHeaders() { + return fieldSetFlags()[19]; + } + + /** Clears the value of the 'headers' field */ + public Builder clearHeaders() { + headers = null; + fieldSetFlags()[19] = false; + return this; + } + + /** Gets the value of the 'outlinks' field */ + public java.util.Map getOutlinks() { + return outlinks; + } + + /** Sets the value of the 'outlinks' field */ + public Builder setOutlinks(java.util.Map value) { + validate(fields()[20], value); + this.outlinks = value; + fieldSetFlags()[20] = true; + return this; + } + + /** Checks whether the 'outlinks' field has been set */ + public boolean hasOutlinks() { + return fieldSetFlags()[20]; + } + + /** Clears the value of the 'outlinks' field */ + public Builder clearOutlinks() { + outlinks = null; + fieldSetFlags()[20] = false; + return this; + } + + /** Gets the value of the 'inlinks' field */ + public java.util.Map getInlinks() { + return inlinks; + } + + /** Sets the value of the 'inlinks' field */ + public Builder setInlinks(java.util.Map value) { + validate(fields()[21], value); + this.inlinks = value; + fieldSetFlags()[21] = true; + return this; + } + + /** Checks whether the 'inlinks' field has been set */ + public boolean hasInlinks() { + return fieldSetFlags()[21]; + } + + /** Clears the value of the 'inlinks' field */ + public Builder clearInlinks() { + inlinks = null; + fieldSetFlags()[21] = false; + return this; + } + + /** Gets the value of the 'markers' field */ + public java.util.Map getMarkers() { + return markers; + } + + /** Sets the value of the 'markers' field */ + public Builder setMarkers(java.util.Map value) { + validate(fields()[22], value); + this.markers = value; + fieldSetFlags()[22] = true; + return this; + } + + /** Checks whether the 'markers' field has been set */ + public boolean hasMarkers() { + return fieldSetFlags()[22]; + } + + /** Clears the value of the 'markers' field */ + public Builder clearMarkers() { + markers = null; + fieldSetFlags()[22] = false; + return this; + } + + /** Gets the value of the 'metadata' field */ + public java.util.Map getMetadata() { + return metadata; + } + + /** Sets the value of the 'metadata' field */ + public Builder setMetadata(java.util.Map value) { + validate(fields()[23], value); + this.metadata = value; + fieldSetFlags()[23] = true; + return this; + } + + /** Checks whether the 'metadata' field has been set */ + public boolean hasMetadata() { + return fieldSetFlags()[23]; + } + + /** Clears the value of the 'metadata' field */ + public Builder clearMetadata() { + metadata = null; + fieldSetFlags()[23] = false; + return this; + } + + /** Gets the value of the 'batchId' field */ + public CharSequence getBatchId() { + return batchId; + } + + /** Sets the value of the 'batchId' field */ + public Builder setBatchId(CharSequence value) { + validate(fields()[24], value); + this.batchId = value; + fieldSetFlags()[24] = true; + return this; + } + + /** Checks whether the 'batchId' field has been set */ + public boolean hasBatchId() { + return fieldSetFlags()[24]; + } + + /** Clears the value of the 'batchId' field */ + public Builder clearBatchId() { + batchId = null; + fieldSetFlags()[24] = false; + return this; + } + + @Override + public WebPage build() { + try { + WebPage record = new WebPage(); + record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[4]); + record.baseUrl = fieldSetFlags()[1] ? this.baseUrl : (CharSequence) defaultValue(fields()[1]); + record.status = fieldSetFlags()[2] ? this.status : (Integer) defaultValue(fields()[2]); + record.fetchTime = fieldSetFlags()[3] ? this.fetchTime : (Long) defaultValue(fields()[3]); + record.prevFetchTime = fieldSetFlags()[4] ? this.prevFetchTime : (Long) defaultValue(fields()[4]); + record.fetchInterval = fieldSetFlags()[5] ? this.fetchInterval : (Integer) defaultValue(fields()[5]); + record.retriesSinceFetch = fieldSetFlags()[6] ? this.retriesSinceFetch : (Integer) defaultValue(fields()[6]); + record.modifiedTime = fieldSetFlags()[7] ? this.modifiedTime : (Long) defaultValue(fields()[7]); + record.prevModifiedTime = fieldSetFlags()[8] ? this.prevModifiedTime : (Long) defaultValue(fields()[8]); + record.protocolStatus = fieldSetFlags()[9] ? this.protocolStatus : (ProtocolStatus) defaultValue(fields()[9]); + record.content = fieldSetFlags()[10] ? this.content : (java.nio.ByteBuffer) defaultValue(fields()[10]); + record.contentType = fieldSetFlags()[11] ? this.contentType : (CharSequence) defaultValue(fields()[11]); + record.prevSignature = fieldSetFlags()[12] ? this.prevSignature : (java.nio.ByteBuffer) defaultValue(fields()[12]); + record.signature = fieldSetFlags()[13] ? this.signature : (java.nio.ByteBuffer) defaultValue(fields()[13]); + record.title = fieldSetFlags()[14] ? this.title : (CharSequence) defaultValue(fields()[14]); + record.text = fieldSetFlags()[15] ? this.text : (CharSequence) defaultValue(fields()[15]); + record.parseStatus = fieldSetFlags()[16] ? this.parseStatus : (ParseStatus) defaultValue(fields()[16]); + record.score = fieldSetFlags()[17] ? this.score : (Float) defaultValue(fields()[17]); + record.reprUrl = fieldSetFlags()[18] ? this.reprUrl : (CharSequence) defaultValue(fields()[18]); + record.headers = fieldSetFlags()[19] ? this.headers : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[19])); + record.outlinks = fieldSetFlags()[20] ? this.outlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[20])); + record.inlinks = fieldSetFlags()[21] ? this.inlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[21])); + record.markers = fieldSetFlags()[22] ? this.markers : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[22])); + record.metadata = fieldSetFlags()[23] ? this.metadata : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[23])); + record.batchId = fieldSetFlags()[24] ? this.batchId : (CharSequence) defaultValue(fields()[24]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } } - public Utf8 getBatchId() { - return (Utf8) get(23); + + public Tombstone getTombstone(){ + return TOMBSTONE; } - public void setBatchId(Utf8 value) { - put(23, value); + + public WebPage newInstance(){ + return newBuilder().build(); } -} + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends WebPage implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'baseUrl' field. + */ + public CharSequence getBaseUrl() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'baseUrl' field. + * @param value the value to set. + */ + public void setBaseUrl(CharSequence value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'baseUrl' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isBaseUrlDirty(CharSequence value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'status' field. + */ + public Integer getStatus() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'status' field. + * @param value the value to set. + */ + public void setStatus(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'status' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isStatusDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'fetchTime' field. + */ + public Long getFetchTime() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'fetchTime' field. + * @param value the value to set. + */ + public void setFetchTime(Long value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'fetchTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isFetchTimeDirty(Long value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'prevFetchTime' field. + */ + public Long getPrevFetchTime() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'prevFetchTime' field. + * @param value the value to set. + */ + public void setPrevFetchTime(Long value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'prevFetchTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isPrevFetchTimeDirty(Long value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'fetchInterval' field. + */ + public Integer getFetchInterval() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'fetchInterval' field. + * @param value the value to set. + */ + public void setFetchInterval(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'fetchInterval' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isFetchIntervalDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'retriesSinceFetch' field. + */ + public Integer getRetriesSinceFetch() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'retriesSinceFetch' field. + * @param value the value to set. + */ + public void setRetriesSinceFetch(Integer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'retriesSinceFetch' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isRetriesSinceFetchDirty(Integer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'modifiedTime' field. + */ + public Long getModifiedTime() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'modifiedTime' field. + * @param value the value to set. + */ + public void setModifiedTime(Long value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'modifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isModifiedTimeDirty(Long value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'prevModifiedTime' field. + */ + public Long getPrevModifiedTime() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'prevModifiedTime' field. + * @param value the value to set. + */ + public void setPrevModifiedTime(Long value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'prevModifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isPrevModifiedTimeDirty(Long value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'protocolStatus' field. + */ + public ProtocolStatus getProtocolStatus() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'protocolStatus' field. + * @param value the value to set. + */ + public void setProtocolStatus(ProtocolStatus value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'protocolStatus' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isProtocolStatusDirty(ProtocolStatus value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'content' field. + */ + public java.nio.ByteBuffer getContent() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'content' field. + * @param value the value to set. + */ + public void setContent(java.nio.ByteBuffer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'content' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isContentDirty(java.nio.ByteBuffer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'contentType' field. + */ + public CharSequence getContentType() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'contentType' field. + * @param value the value to set. + */ + public void setContentType(CharSequence value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'contentType' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isContentTypeDirty(CharSequence value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'prevSignature' field. + */ + public java.nio.ByteBuffer getPrevSignature() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'prevSignature' field. + * @param value the value to set. + */ + public void setPrevSignature(java.nio.ByteBuffer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'prevSignature' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isPrevSignatureDirty(java.nio.ByteBuffer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'signature' field. + */ + public java.nio.ByteBuffer getSignature() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'signature' field. + * @param value the value to set. + */ + public void setSignature(java.nio.ByteBuffer value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'signature' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isSignatureDirty(java.nio.ByteBuffer value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'title' field. + */ + public CharSequence getTitle() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'title' field. + * @param value the value to set. + */ + public void setTitle(CharSequence value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'title' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isTitleDirty(CharSequence value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'text' field. + */ + public CharSequence getText() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'text' field. + * @param value the value to set. + */ + public void setText(CharSequence value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'text' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isTextDirty(CharSequence value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'parseStatus' field. + */ + public ParseStatus getParseStatus() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'parseStatus' field. + * @param value the value to set. + */ + public void setParseStatus(ParseStatus value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'parseStatus' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isParseStatusDirty(ParseStatus value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'score' field. + */ + public Float getScore() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'score' field. + * @param value the value to set. + */ + public void setScore(Float value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'score' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isScoreDirty(Float value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'reprUrl' field. + */ + public CharSequence getReprUrl() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'reprUrl' field. + * @param value the value to set. + */ + public void setReprUrl(CharSequence value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'reprUrl' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isReprUrlDirty(CharSequence value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'headers' field. + */ + public java.util.Map getHeaders() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'headers' field. + * @param value the value to set. + */ + public void setHeaders(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'headers' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isHeadersDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'outlinks' field. + */ + public java.util.Map getOutlinks() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'outlinks' field. + * @param value the value to set. + */ + public void setOutlinks(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isOutlinksDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'inlinks' field. + */ + public java.util.Map getInlinks() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'inlinks' field. + * @param value the value to set. + */ + public void setInlinks(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isInlinksDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'markers' field. + */ + public java.util.Map getMarkers() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'markers' field. + * @param value the value to set. + */ + public void setMarkers(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'markers' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMarkersDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'metadata' field. + */ + public java.util.Map getMetadata() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'metadata' field. + * @param value the value to set. + */ + public void setMetadata(java.util.Map value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isMetadataDirty(java.util.Map value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'batchId' field. + */ + public CharSequence getBatchId() { + throw new UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'batchId' field. + * @param value the value to set. + */ + public void setBatchId(CharSequence value) { + throw new UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'batchId' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isBatchIdDirty(CharSequence value) { + throw new UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + } +} \ No newline at end of file diff --git src/java/org/apache/nutch/tools/DmozParser.java src/java/org/apache/nutch/tools/DmozParser.java index 9d9dc77..8c759c3 100644 --- src/java/org/apache/nutch/tools/DmozParser.java +++ src/java/org/apache/nutch/tools/DmozParser.java @@ -195,11 +195,11 @@ public class DmozParser { if(row!=null){ if (desc.length() > 0) { - row.putToMetadata(new Utf8("_dmoz_desc_"), ByteBuffer.wrap(desc.toString().getBytes())); + row.getMetadata().put(new Utf8("_dmoz_desc_"), ByteBuffer.wrap(desc.toString().getBytes())); desc.delete(0, desc.length()); } if (title.length() > 0) { - row.putToMetadata(new Utf8("_dmoz_title_"), ByteBuffer.wrap(title.toString().getBytes())); + row.getMetadata().put(new Utf8("_dmoz_title_"), ByteBuffer.wrap(title.toString().getBytes())); title.delete(0, title.length()); } store.put(reversedUrl, row); diff --git src/java/org/apache/nutch/util/EncodingDetector.java src/java/org/apache/nutch/util/EncodingDetector.java index 756c1d2..8c5324b 100644 --- src/java/org/apache/nutch/util/EncodingDetector.java +++ src/java/org/apache/nutch/util/EncodingDetector.java @@ -16,6 +16,15 @@ */ package org.apache.nutch.util; +import com.ibm.icu.text.CharsetDetector; +import com.ibm.icu.text.CharsetMatch; +import org.apache.avro.util.Utf8; +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.net.protocols.Response; +import org.apache.nutch.storage.WebPage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayInputStream; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -24,16 +33,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.nutch.net.protocols.Response; -import org.apache.nutch.storage.WebPage; - -import com.ibm.icu.text.CharsetDetector; -import com.ibm.icu.text.CharsetMatch; - /** * A simple class for detecting character encodings. * @@ -165,10 +164,10 @@ public class EncodingDetector { public void autoDetectClues(WebPage page, boolean filter) { autoDetectClues(page.getContent(), page.getContentType(), - parseCharacterEncoding(page.getFromHeaders(CONTENT_TYPE_UTF8)), filter); + parseCharacterEncoding(page.getHeaders().get(CONTENT_TYPE_UTF8)), filter); } - private void autoDetectClues(ByteBuffer dataBuffer, Utf8 typeUtf8, + private void autoDetectClues(ByteBuffer dataBuffer, CharSequence typeUtf8, String encoding, boolean filter) { int length = dataBuffer.remaining(); String type = TableUtil.toString(typeUtf8); @@ -224,7 +223,7 @@ public class EncodingDetector { * @return Guessed encoding or defaultValue */ public String guessEncoding(WebPage page, String defaultValue) { - Utf8 baseUrlUtf8 = page.getBaseUrl(); + CharSequence baseUrlUtf8 = page.getBaseUrl(); String baseUrl = TableUtil.toString(baseUrlUtf8); return guessEncoding(baseUrl, defaultValue); } @@ -347,9 +346,9 @@ public class EncodingDetector { * This method was copied from org.apache.catalina.util.RequestUtil, * which is licensed under the Apache License, Version 2.0 (the "License"). * - * @param contentType a content type header + * @param contentTypeUtf8 */ - public static String parseCharacterEncoding(Utf8 contentTypeUtf8) { + public static String parseCharacterEncoding(CharSequence contentTypeUtf8) { if (contentTypeUtf8 == null) return (null); String contentType = contentTypeUtf8.toString(); diff --git src/java/org/apache/nutch/util/NutchJob.java src/java/org/apache/nutch/util/NutchJob.java index cdb8f6a..6121c1c 100644 --- src/java/org/apache/nutch/util/NutchJob.java +++ src/java/org/apache/nutch/util/NutchJob.java @@ -17,14 +17,14 @@ package org.apache.nutch.util; -import java.io.IOException; - import org.apache.avro.util.Utf8; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Job; import org.apache.nutch.metadata.Nutch; +import java.io.IOException; + /** A {@link Job} for Nutch jobs. */ public class NutchJob extends Job { @@ -58,7 +58,7 @@ public class NutchJob extends Job { return succeeded; } - public static boolean shouldProcess(Utf8 mark, Utf8 batchId) { + public static boolean shouldProcess(CharSequence mark, Utf8 batchId) { if (mark == null) { return false; } diff --git src/java/org/apache/nutch/util/TableUtil.java src/java/org/apache/nutch/util/TableUtil.java index efe7b59..2b7c755 100644 --- src/java/org/apache/nutch/util/TableUtil.java +++ src/java/org/apache/nutch/util/TableUtil.java @@ -16,14 +16,13 @@ ******************************************************************************/ package org.apache.nutch.util; +import org.apache.avro.util.Utf8; +import org.apache.commons.lang.StringUtils; + import java.net.MalformedURLException; import java.net.URL; import java.nio.ByteBuffer; -import org.apache.avro.util.Utf8; -import org.apache.commons.lang.StringUtils; -import org.apache.nutch.util.StringUtil; - public class TableUtil { public static final ByteBuffer YES_VAL = ByteBuffer.wrap(new byte[] { 'y' }); @@ -148,11 +147,12 @@ public class TableUtil { * Convert given Utf8 instance to String and and cleans out * any offending "�" from the String. * + * * @param utf8 * Utf8 object * @return string-ifed Utf8 object or null if Utf8 instance is null */ - public static String toString(Utf8 utf8) { + public static String toString(CharSequence utf8) { return (utf8 == null ? null : StringUtil.cleanField(utf8.toString())); } diff --git src/java/org/apache/nutch/util/WebPageWritable.java src/java/org/apache/nutch/util/WebPageWritable.java index d4d26bd..68f289c 100644 --- src/java/org/apache/nutch/util/WebPageWritable.java +++ src/java/org/apache/nutch/util/WebPageWritable.java @@ -16,15 +16,15 @@ ******************************************************************************/ package org.apache.nutch.util; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - +import org.apache.gora.util.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Writable; import org.apache.nutch.storage.WebPage; -import org.apache.gora.util.IOUtils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; public class WebPageWritable extends Configured implements Writable { @@ -32,7 +32,7 @@ implements Writable { private WebPage webPage; public WebPageWritable() { - this(null, new WebPage()); + this(null, WebPage.newBuilder().build()); } public WebPageWritable(Configuration conf, WebPage webPage) { diff --git src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java index b9c8319..7d4a1bd 100644 --- src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java +++ src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java @@ -17,16 +17,7 @@ package org.creativecommons.nutch; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.HashSet; -import java.util.StringTokenizer; - import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.IndexingException; import org.apache.nutch.indexer.IndexingFilter; @@ -35,6 +26,15 @@ import org.apache.nutch.metadata.CreativeCommons; import org.apache.nutch.storage.WebPage; import org.apache.nutch.storage.WebPage.Field; import org.apache.nutch.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.StringTokenizer; /** Adds basic searchable fields to a document. */ public class CCIndexingFilter implements IndexingFilter { @@ -100,7 +100,7 @@ public class CCIndexingFilter implements IndexingFilter { public NutchDocument filter(NutchDocument doc, String url, WebPage page) throws IndexingException { - ByteBuffer blicense = page.getFromMetadata(new Utf8( + ByteBuffer blicense = page.getMetadata().get(new Utf8( CreativeCommons.LICENSE_URL)); if (blicense != null) { String licenseUrl = Bytes.toString(blicense); @@ -117,7 +117,7 @@ public class CCIndexingFilter implements IndexingFilter { } // index the license location as cc:meta=xxx - ByteBuffer blicenseloc = page.getFromMetadata(new Utf8( + ByteBuffer blicenseloc = page.getMetadata().get(new Utf8( CreativeCommons.LICENSE_LOCATION)); if (blicenseloc != null) { String licenseLocation = Bytes.toString(blicenseloc); @@ -125,7 +125,7 @@ public class CCIndexingFilter implements IndexingFilter { } // index the work type cc:type=xxx - ByteBuffer bworkType = page.getFromMetadata(new Utf8( + ByteBuffer bworkType = page.getMetadata().get(new Utf8( CreativeCommons.WORK_TYPE)); if (bworkType != null) { String workType = Bytes.toString(bworkType); diff --git src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java index 8223a6e..dc896f7 100644 --- src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java +++ src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java @@ -17,6 +17,19 @@ package org.creativecommons.nutch; +import org.apache.avro.util.Utf8; +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.metadata.CreativeCommons; +import org.apache.nutch.parse.*; +import org.apache.nutch.storage.WebPage; +import org.apache.nutch.storage.WebPage.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.*; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import java.io.StringReader; import java.net.MalformedURLException; import java.net.URL; @@ -25,29 +38,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.nutch.metadata.CreativeCommons; -import org.apache.nutch.parse.HTMLMetaTags; -import org.apache.nutch.parse.ParseFilter; -import org.apache.nutch.parse.Parse; -import org.apache.nutch.parse.ParseException; -import org.apache.nutch.parse.ParseStatusUtils; -import org.apache.nutch.storage.WebPage; -import org.apache.nutch.storage.WebPage.Field; -import org.w3c.dom.Comment; -import org.w3c.dom.Document; -import org.w3c.dom.DocumentFragment; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; - /** Adds metadata identifying the Creative Commons license used, if any. */ public class CCParseFilter implements ParseFilter { public static final Logger LOG = LoggerFactory.getLogger(CCParseFilter.class); @@ -95,9 +85,9 @@ public class CCParseFilter implements ParseFilter { if (LOG.isDebugEnabled()) { LOG.debug("CC: found " + licenseUrl + " in " + licenseLocation + " of " + base); } - page.putToMetadata(new Utf8(CreativeCommons.LICENSE_URL), + page.getMetadata().put(new Utf8(CreativeCommons.LICENSE_URL), ByteBuffer.wrap(licenseUrl.getBytes())); - page.putToMetadata(new Utf8(CreativeCommons.LICENSE_LOCATION), + page.getMetadata().put(new Utf8(CreativeCommons.LICENSE_LOCATION), ByteBuffer.wrap(licenseLocation.getBytes())); } @@ -105,7 +95,7 @@ public class CCParseFilter implements ParseFilter { if (LOG.isDebugEnabled()) { LOG.debug("CC: found " + walker.workType + " in " + base); } - page.putToMetadata(new Utf8(CreativeCommons.WORK_TYPE), + page.getMetadata().put(new Utf8(CreativeCommons.WORK_TYPE), ByteBuffer.wrap(walker.workType.getBytes())); } diff --git src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java index a64bf46..09c506e 100755 --- src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java +++ src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java @@ -17,18 +17,22 @@ package org.creativecommons.nutch; -import org.apache.nutch.parse.ParseUtil; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.parse.ParseUtil; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.Bytes; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import java.io.*; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; import java.nio.ByteBuffer; -import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class TestCCParseFilter { @@ -64,7 +68,7 @@ public class TestCCParseFilter { byte[] bytes = out.toByteArray(); Configuration conf = NutchConfiguration.create(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(url)); page.setContent(ByteBuffer.wrap(bytes)); MimeUtil mimeutil = new MimeUtil(conf); @@ -73,11 +77,11 @@ public class TestCCParseFilter { new ParseUtil(conf).parse(url, page); - ByteBuffer bb = page.getFromMetadata(new Utf8("License-Url")); + ByteBuffer bb = page.getMetadata().get(new Utf8("License-Url")); assertEquals(license, Bytes.toString(bb)); - bb = page.getFromMetadata(new Utf8("License-Location")); + bb = page.getMetadata().get(new Utf8("License-Location")); assertEquals(location, Bytes.toString(bb)); - bb = page.getFromMetadata(new Utf8("Work-Type")); + bb = page.getMetadata().get(new Utf8("Work-Type")); assertEquals(type, Bytes.toString(bb)); } } diff --git src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java index 70a15c6..2731c0b 100644 --- src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java +++ src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java @@ -16,10 +16,6 @@ */ package org.apache.nutch.indexer.anchor; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map.Entry; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.IndexingException; @@ -30,6 +26,11 @@ import org.apache.nutch.util.TableUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.CharSequence; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map.Entry; + /** * Indexing filter that offers an option to either index all inbound anchor text for * a document or deduplicate anchors. Deduplication does have it's con's, @@ -82,7 +83,7 @@ public class AnchorIndexingFilter implements IndexingFilter { throws IndexingException { HashSet set = null; - for (Entry e : page.getInlinks().entrySet()) { + for (Entry e : page.getInlinks().entrySet()) { String anchor = TableUtil.toString(e.getValue()); if(anchor.equals("")) diff --git src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java index 78d3e8a..84c1d41 100644 --- src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java +++ src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java @@ -39,10 +39,10 @@ public class TestAnchorIndexingFilter { AnchorIndexingFilter filter = new AnchorIndexingFilter(); filter.setConf(conf); NutchDocument doc = new NutchDocument(); - WebPage page = new WebPage(); - page.putToInlinks(new Utf8("http://example1.com/"), new Utf8("cool site")); - page.putToInlinks(new Utf8("http://example2.com/"), new Utf8("cool site")); - page.putToInlinks(new Utf8("http://example3.com/"), new Utf8("fun site")); + WebPage page = WebPage.newBuilder().build(); + page.getInlinks().put(new Utf8("http://example1.com/"), new Utf8("cool site")); + page.getInlinks().put(new Utf8("http://example2.com/"), new Utf8("cool site")); + page.getInlinks().put(new Utf8("http://example3.com/"), new Utf8("fun site")); filter.filter(doc, "http://myurldoesnotmatter.com/", page); assertTrue("test if there is an anchor at all", doc.getFieldNames().contains("anchor")); diff --git src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java index 87fbefa..f5b2ec1 100644 --- src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java +++ src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java @@ -17,15 +17,6 @@ package org.apache.nutch.indexer.basic; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Date; -import java.util.HashSet; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.IndexingException; import org.apache.nutch.indexer.IndexingFilter; @@ -35,6 +26,15 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.Bytes; import org.apache.nutch.util.TableUtil; import org.apache.solr.common.util.DateUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; /** Adds basic searchable fields to a document. The fields are: * host - add host as un-stored, indexed and tokenized @@ -74,9 +74,9 @@ public class BasicIndexingFilter implements IndexingFilter { throws IndexingException { String reprUrl = null; - if (page.isReadable(WebPage.Field.REPR_URL.getIndex())) { +// if (page.isReadable(WebPage.Field.REPR_URL.getIndex())) { reprUrl = TableUtil.toString(page.getReprUrl()); - } +// } String host = null; try { @@ -118,7 +118,7 @@ public class BasicIndexingFilter implements IndexingFilter { } // add cached content/summary display policy, if available ByteBuffer cachingRaw = page - .getFromMetadata(Nutch.CACHING_FORBIDDEN_KEY_UTF8); + .getMetadata().get(Nutch.CACHING_FORBIDDEN_KEY_UTF8); String caching = Bytes.toString(cachingRaw); if (caching != null && !caching.equals(Nutch.CACHING_FORBIDDEN_NONE)) { doc.add("cache", caching); diff --git src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java index c20b14e..66c8cdd 100644 --- src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java +++ src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java @@ -16,8 +16,6 @@ */ package org.apache.nutch.indexer.basic; -import java.nio.ByteBuffer; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.NutchDocument; @@ -25,6 +23,9 @@ import org.apache.nutch.metadata.Nutch; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; import org.junit.Test; + +import java.nio.ByteBuffer; + import static org.junit.Assert.*; /** @@ -47,13 +48,13 @@ public class TestBasicIndexingFilter { filter.setConf(conf); assertNotNull(filter); NutchDocument doc = new NutchDocument(); - WebPage page = new WebPage(); - page.putToInlinks(new Utf8("http://nutch.apache.org/"), new Utf8("Welcome to Nutch")); + WebPage page = WebPage.newBuilder().build(); + page.getInlinks().put(new Utf8("http://nutch.apache.org/"), new Utf8("Welcome to Nutch")); page.setTitle(new Utf8("Welcome to Nutch")); page.setReprUrl(new Utf8("http://www.urldoesnotmatter.org")); byte[] bytes = new byte[10]; ByteBuffer bbuf = ByteBuffer.wrap(bytes); - page.putToMetadata(Nutch.CACHING_FORBIDDEN_KEY_UTF8, bbuf); + page.getMetadata().put(Nutch.CACHING_FORBIDDEN_KEY_UTF8, bbuf); page.setFetchTime(System.currentTimeMillis()); try { filter.filter(doc, "http://www.apache.org/", page); @@ -79,8 +80,8 @@ public class TestBasicIndexingFilter { filter.setConf(conf); assertNotNull(filter); NutchDocument doc = new NutchDocument(); - WebPage page = new WebPage(); - page.putToInlinks(new Utf8("http://exceedmaximumtitleurl.org/"), new Utf8("exceeding title site")); + WebPage page = WebPage.newBuilder().build(); + page.getInlinks().put(new Utf8("http://exceedmaximumtitleurl.org/"), new Utf8("exceeding title site")); page.setTitle(new Utf8("This title exceeds maximum characters")); try { filter.filter(doc, "http://www.apache.org/", page); diff --git src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java index 146d4a6..7ec70c1 100644 --- src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java +++ src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java @@ -52,7 +52,7 @@ public class MetadataIndexer implements IndexingFilter { // add the fields from parsemd if (parseFieldnames != null) { for (String metatag : parseFieldnames) { - ByteBuffer bvalues = page.getFromMetadata(new Utf8(PARSE_META_PREFIX + ByteBuffer bvalues = page.getMetadata().get(new Utf8(PARSE_META_PREFIX + metatag)); if (bvalues != null) { String value = new String(bvalues.array()); diff --git src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java index c95a0ee..3a0bba3 100644 --- src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java +++ src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java @@ -1,19 +1,3 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.nutch.indexer.more; import java.text.ParseException; @@ -84,8 +68,8 @@ public class MoreIndexingFilter implements IndexingFilter { // last-modified, or, if that's not present, use fetch time. private NutchDocument addTime(NutchDocument doc, WebPage page, String url) { long time = -1; - Utf8 lastModified = page - .getFromHeaders(new Utf8(HttpHeaders.LAST_MODIFIED)); + CharSequence lastModified = page + .getHeaders().get(new Utf8(HttpHeaders.LAST_MODIFIED)); // String lastModified = data.getMeta(Metadata.LAST_MODIFIED); if (lastModified != null) { // try parse last-modified time = getTime(lastModified.toString(), url); // use as time @@ -139,8 +123,8 @@ public class MoreIndexingFilter implements IndexingFilter { // Add Content-Length private NutchDocument addLength(NutchDocument doc, WebPage page, String url) { - Utf8 contentLength = page.getFromHeaders(new Utf8( - HttpHeaders.CONTENT_LENGTH)); + CharSequence contentLength = page.getHeaders().get(new Utf8( + HttpHeaders.CONTENT_LENGTH)); if (contentLength != null) { // NUTCH-1010 ContentLength not trimmed String trimmed = contentLength.toString().trim(); @@ -173,9 +157,9 @@ public class MoreIndexingFilter implements IndexingFilter { */ private NutchDocument addType(NutchDocument doc, WebPage page, String url) { String mimeType = null; - Utf8 contentType = page.getContentType(); + CharSequence contentType = page.getContentType(); if (contentType == null) - contentType = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_TYPE)); + contentType = page.getHeaders().get(new Utf8(HttpHeaders.CONTENT_TYPE)); if (contentType == null) { // Note by Jerome Charron on 20050415: // Content Type not solved by a previous plugin @@ -249,7 +233,7 @@ public class MoreIndexingFilter implements IndexingFilter { } private NutchDocument resetTitle(NutchDocument doc, WebPage page, String url) { - Utf8 contentDisposition = page.getFromHeaders(new Utf8( + CharSequence contentDisposition = page.getHeaders().get(new Utf8( HttpHeaders.CONTENT_DISPOSITION)); if (contentDisposition == null) return doc; diff --git src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java index 3cf7466..95dd61d 100644 --- src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java +++ src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java @@ -16,11 +16,6 @@ */ package org.apache.nutch.indexer.more; -import java.nio.ByteBuffer; - -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.IndexingException; @@ -28,6 +23,11 @@ import org.apache.nutch.indexer.NutchDocument; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.EncodingDetector; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.*; public class TestMoreIndexingFilter { @@ -56,7 +56,7 @@ public class TestMoreIndexingFilter { assertNotNull(filter); NutchDocument doc = new NutchDocument(); try{ - filter.filter(doc, "http://nutch.apache.org/index.html", new WebPage()); + filter.filter(doc, "http://nutch.apache.org/index.html", WebPage.newBuilder().build()); } catch(Exception e){ e.printStackTrace(); @@ -78,11 +78,11 @@ public class TestMoreIndexingFilter { private void assertContentType(Configuration conf, String source, String expected) throws IndexingException { MoreIndexingFilter filter = new MoreIndexingFilter(); filter.setConf(conf); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); String url = "http://www.example.com/"; page.setContent(ByteBuffer.wrap("text".getBytes())); page.setTitle(new Utf8("title")); - page.putToHeaders(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8(source)); + page.getHeaders().put(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8(source)); NutchDocument doc = filter.filter(new NutchDocument(), url, page); assertEquals("mime type not detected", expected, doc.getFieldValue("type")); } diff --git src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java index 2b64a2a..995908b 100644 --- src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java +++ src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java @@ -17,33 +17,30 @@ package org.apache.nutch.analysis.lang; // JDK imports -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.parse.HTMLMetaTags; -import org.apache.nutch.parse.ParseFilter; import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseFilter; import org.apache.nutch.storage.WebPage; import org.apache.nutch.storage.WebPage.Field; import org.apache.nutch.util.Bytes; import org.apache.nutch.util.NodeWalker; import org.apache.tika.language.LanguageIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.w3c.dom.DocumentFragment; import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; +import java.lang.CharSequence; +import java.nio.ByteBuffer; +import java.util.*; + /** * Adds metadata identifying language of document if found We could also run * statistical analysis here but we'd miss all other formats @@ -116,8 +113,8 @@ public class HTMLLanguageParser implements ParseFilter { } if (lang != null) { - page.putToMetadata(new Utf8(Metadata.LANGUAGE), ByteBuffer.wrap(lang - .getBytes())); + page.getMetadata().put(new Utf8(Metadata.LANGUAGE), ByteBuffer.wrap(lang + .getBytes())); return parse; } @@ -138,7 +135,7 @@ public class HTMLLanguageParser implements ParseFilter { return lang; } - Utf8 ulang = page.getFromHeaders(new Utf8(Response.CONTENT_LANGUAGE)); + CharSequence ulang = page.getHeaders().get(new Utf8(Response.CONTENT_LANGUAGE)); if (ulang != null) { lang = ulang.toString(); } @@ -176,7 +173,7 @@ public class HTMLLanguageParser implements ParseFilter { // Check in the metadata whether the language has already been stored there by // Tika private static ByteBuffer getLanguageFromMetadata( - Map metadata) { + Map metadata) { if (metadata == null) return null; diff --git src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java index f72c39d..dbe2953 100644 --- src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java +++ src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java @@ -17,9 +17,6 @@ package org.apache.nutch.analysis.lang; // Nutch imports -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.HashSet; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -31,6 +28,10 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.storage.WebPage.Field; import org.apache.nutch.util.Bytes; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; + /** * An {@link org.apache.nutch.indexer.IndexingFilter} that adds a * lang (language) field to the document. @@ -61,7 +62,7 @@ public class LanguageIndexingFilter implements IndexingFilter { throws IndexingException { // check if LANGUAGE found, possibly put there by HTMLLanguageParser - ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE)); + ByteBuffer blang = page.getMetadata().get(new Utf8(Metadata.LANGUAGE)); String lang = Bytes.toString(blang); if (lang == null || lang.length() == 0) { diff --git src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java index e444b17..578a4f1 100644 --- src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java +++ src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java @@ -17,12 +17,6 @@ package org.apache.nutch.analysis.lang; // JUnit imports -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.nio.ByteBuffer; - -import org.junit.Test; -import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.nutch.metadata.Metadata; @@ -32,6 +26,14 @@ import org.apache.nutch.util.Bytes; import org.apache.nutch.util.EncodingDetector; import org.apache.nutch.util.NutchConfiguration; import org.apache.tika.language.LanguageIdentifier; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TestHTMLLanguageParser { @@ -59,7 +61,7 @@ public class TestHTMLLanguageParser { for (int t = 0; t < docs.length; t++) { WebPage page = getPage(docs[t]); parser.parse(URL.toString(), page); - ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE)); + ByteBuffer blang = page.getMetadata().get(new Utf8(Metadata.LANGUAGE)); String lang = Bytes.toString(blang); assertEquals(metalanguages[t], lang); } @@ -145,12 +147,12 @@ public class TestHTMLLanguageParser { } private WebPage getPage(String text) { - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(BASE); page.setContent(ByteBuffer.wrap(text.getBytes())); page.setContentType(new Utf8("text/html")); page - .putToHeaders(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/html")); + .getHeaders().put(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/html")); return page; } } diff --git src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java index 6abfadc..69ea196 100644 --- src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java +++ src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java @@ -142,7 +142,7 @@ public abstract class HttpBase implements Protocol { int elapsedTime =(int) (System.currentTimeMillis() - startTime); if(this.responseTime) { - page.putToMetadata(RESPONSE_TIME, ByteBuffer.wrap(Bytes.toBytes(elapsedTime))); + page.getMetadata().put(RESPONSE_TIME, ByteBuffer.wrap(Bytes.toBytes(elapsedTime))); } int code = response.getCode(); @@ -373,7 +373,7 @@ public abstract class HttpBase implements Protocol { url = args[i]; } - ProtocolOutput out = http.getProtocolOutput(url, new WebPage()); + ProtocolOutput out = http.getProtocolOutput(url, WebPage.newBuilder().build()); Content content = out.getContent(); System.out.println("Status: " + out.getStatus()); diff --git src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java index dd6d412..df60500 100644 --- src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java +++ src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java @@ -17,19 +17,17 @@ package org.apache.nutch.protocol.http.api; -import java.net.URL; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import crawlercommons.robots.BaseRobotRules; +import crawlercommons.robots.SimpleRobotRules; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.protocol.Protocol; import org.apache.nutch.protocol.RobotRulesParser; import org.apache.nutch.storage.WebPage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import crawlercommons.robots.BaseRobotRules; -import crawlercommons.robots.SimpleRobotRules; +import java.net.URL; /** * This class is used for parsing robots for urls belonging to HTTP protocol. @@ -73,7 +71,7 @@ public class HttpRobotRulesParser extends RobotRulesParser { if (LOG.isTraceEnabled()) { LOG.trace("cache miss " + url); } try { Response response = ((HttpBase)http).getResponse(new URL(url, "/robots.txt"), - new WebPage(), true); + WebPage.newBuilder().build(), true); // try one level of redirection ? if (response.getCode() == 301 || response.getCode() == 302) { String redirection = response.getHeader("Location"); @@ -89,7 +87,7 @@ public class HttpRobotRulesParser extends RobotRulesParser { redir = new URL(redirection); } - response = ((HttpBase)http).getResponse(redir, new WebPage(), true); + response = ((HttpBase)http).getResponse(redir, WebPage.newBuilder().build(), true); } } diff --git src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java index 9cc268d..a1dba9d 100644 --- src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java +++ src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java @@ -85,7 +85,7 @@ public class RelTagIndexingFilter implements IndexingFilter { @Override public NutchDocument filter(NutchDocument doc, String url, WebPage page) throws IndexingException { // Check if some Rel-Tags found, possibly put there by RelTagParser - ByteBuffer bb = page.getFromMetadata(new Utf8(RelTagParser.REL_TAG)); + ByteBuffer bb = page.getMetadata().get(new Utf8(RelTagParser.REL_TAG)); if (bb != null) { String[] tags = Bytes.toString(bb).split("\t"); diff --git src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java index 7ec23ef..385d3b8 100644 --- src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java +++ src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java @@ -171,7 +171,7 @@ public class RelTagParser implements ParseFilter { sb.append("\t"); } ByteBuffer bb = ByteBuffer.wrap(sb.toString().getBytes()); - page.putToMetadata(new Utf8(REL_TAG), bb); + page.getMetadata().put(new Utf8(REL_TAG), bb); return parse; } } diff --git src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java index 8634b8a..4656a8b 100644 --- src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java +++ src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java @@ -16,14 +16,15 @@ */ package org.apache.nutch.microformats.reltag; -import java.nio.ByteBuffer; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.NutchDocument; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; import org.junit.Test; + +import java.nio.ByteBuffer; + import static org.junit.Assert.*; /** @@ -42,10 +43,10 @@ import static org.junit.Assert.*; filter.setConf(conf); assertNotNull(filter); NutchDocument doc = new NutchDocument(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); byte[] bytes = new byte[10]; ByteBuffer bbuf = ByteBuffer.wrap(bytes); - page.putToMetadata(new Utf8(RelTagParser.REL_TAG), bbuf); + page.getMetadata().put(new Utf8(RelTagParser.REL_TAG), bbuf); try { filter.filter(doc, "http://nutch.apache.org/", page); } catch (Exception e) { diff --git src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java index 1206258..923683a 100644 --- src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java +++ src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java @@ -16,12 +16,6 @@ */ package org.apache.nutch.microformats.reltag; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.parse.Parse; @@ -35,6 +29,14 @@ import org.junit.Test; import static org.junit.Assert.*; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; + /** * Junit test for {@link RelTagParser} based mainly John Xing's parser tests. * We are not concerned with actual parse text within the sample file, instead @@ -76,7 +78,7 @@ public class TestRelTagParser { in.readFully(bytes); in.close(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(urlString)); page.setContent(ByteBuffer.wrap(bytes)); MimeUtil mimeutil = new MimeUtil(conf); @@ -84,7 +86,7 @@ public class TestRelTagParser { page.setContentType(new Utf8(mtype)); parse = new ParseUtil(conf).parse(urlString, page); //begin assertion for tests - ByteBuffer bbuf = page.getFromMetadata(new Utf8("Rel-Tag")); + ByteBuffer bbuf = page.getMetadata().get(new Utf8("Rel-Tag")); byte[] byteArray = new byte[bbuf.remaining()]; bbuf.get(byteArray); String s = new String(byteArray); diff --git src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java index 5c38ee2..a04aeda 100644 --- src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java +++ src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java @@ -241,19 +241,19 @@ public class HtmlParser implements Parser { } } - ParseStatus status = new ParseStatus(); - status.setMajorCode(ParseStatusCodes.SUCCESS); + ParseStatus status = ParseStatus.newBuilder().build(); + status.setMajorCode((int)ParseStatusCodes.SUCCESS); if (metaTags.getRefresh()) { - status.setMinorCode(ParseStatusCodes.SUCCESS_REDIRECT); - status.addToArgs(new Utf8(metaTags.getRefreshHref().toString())); - status.addToArgs(new Utf8(Integer.toString(metaTags.getRefreshTime()))); + status.setMinorCode((int)ParseStatusCodes.SUCCESS_REDIRECT); + status.getArgs().add(new Utf8(metaTags.getRefreshHref().toString())); + status.getArgs().add(new Utf8(Integer.toString(metaTags.getRefreshTime()))); } Parse parse = new Parse(text, title, outlinks, status); parse = htmlParseFilters.filter(url, page, parse, metaTags, root); if (metaTags.getNoCache()) { // not okay to cache - page.putToMetadata(new Utf8(Nutch.CACHING_FORBIDDEN_KEY), + page.getMetadata().put(new Utf8(Nutch.CACHING_FORBIDDEN_KEY), ByteBuffer.wrap(Bytes.toBytes(cachingPolicy))); } @@ -352,7 +352,7 @@ public class HtmlParser implements Parser { Configuration conf = NutchConfiguration.create(); HtmlParser parser = new HtmlParser(); parser.setConf(conf); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(url)); page.setContent(ByteBuffer.wrap(bytes)); page.setContentType(new Utf8("text/html")); diff --git src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java index c540b70..caf7487 100644 --- src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java +++ src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java @@ -135,8 +135,7 @@ public class TestHtmlParser { String text = parse.getText(); String title = parse.getTitle(); //String keywords = parse.getMeta("keywords"); - String keywords = Bytes.toString(page - .getFromMetadata(new Utf8("keywords"))); + String keywords = Bytes.toString(page.getMetadata().get(new Utf8("keywords"))); LOG.info(name); LOG.info("title:\t" + title); LOG.info("keywords:\t" + keywords); diff --git src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java index 7386218..3486481 100644 --- src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java +++ src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java @@ -16,12 +16,6 @@ */ package org.apache.nutch.parse.js; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.parse.Outlink; @@ -34,7 +28,14 @@ import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; /** * JUnit test case for {@link JSParseFilter} which tests @@ -74,7 +75,7 @@ public class TestJSParseFilter { dip.readFully(bytes); dip.close(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(urlString)); page.setContent(ByteBuffer.wrap(bytes)); MimeUtil mutil = new MimeUtil(conf); diff --git src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java index 5a5a126..365b879 100644 --- src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java +++ src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java @@ -77,10 +77,9 @@ public class MetaTagsParser implements ParseFilter { // check in the metadata first : the tika-parser // might have stored the values there already - Iterator> iterator = page.getMetadata().entrySet() - .iterator(); + Iterator> iterator = page.getMetadata().entrySet().iterator(); while (iterator.hasNext()) { - Entry entry = iterator.next(); + Entry entry = iterator.next(); String mdName = entry.getKey().toString(); String value = Bytes.toStringBinary(entry.getValue()); if (metatagset.contains("*") || metatagset.contains(mdName.toLowerCase())) { @@ -94,7 +93,7 @@ public class MetaTagsParser implements ParseFilter { Iterator> itm = metadata.entrySet().iterator(); while (iterator.hasNext()) { Entry entry = itm.next(); - page.putToMetadata(entry.getKey(), entry.getValue()); + page.getMetadata().put(entry.getKey(), entry.getValue()); } Properties generalMetaTags = metaTags.getGeneralTags(); @@ -118,7 +117,7 @@ public class MetaTagsParser implements ParseFilter { if (metatagset.contains("*") || metatagset.contains(name.toLowerCase())) { // Add the recently parsed value of multiValued array to metadata LOG.debug("Found meta tag : " + name + "\t" + sb.toString()); - page.putToMetadata(new Utf8(PARSE_META_PREFIX + name.toLowerCase()), + page.getMetadata().put(new Utf8(PARSE_META_PREFIX + name.toLowerCase()), ByteBuffer.wrap(Bytes.toBytes(sb.toString()))); } } @@ -132,7 +131,7 @@ public class MetaTagsParser implements ParseFilter { // specified * if (metatagset.contains("*") || metatagset.contains(name.toLowerCase())) { LOG.debug("Found meta tag : " + name + "\t" + value); - page.putToMetadata(new Utf8(PARSE_META_PREFIX + name.toLowerCase()), + page.getMetadata().put(new Utf8(PARSE_META_PREFIX + name.toLowerCase()), ByteBuffer.wrap(value.getBytes())); } } diff --git src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java index 7b63f9f..ad8e5bc 100644 --- src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java +++ src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java @@ -58,13 +58,14 @@ public class TestMetaTagsParser { /** * + * * @param fileName * This variable set test file. * @param useUtil * If value is True method use ParseUtil * @return If successfully document parsed, it return metatags */ - public Map parseMetaTags(String fileName, boolean useUtil) { + public Map parseMetaTags(String fileName, boolean useUtil) { try { Configuration conf = NutchConfiguration.create(); String urlString = "file:" + sampleDir + fileSeparator + fileName; @@ -185,7 +186,7 @@ public class TestMetaTagsParser { @Test public void testMetaTagsParserWithConf() { // check that we get the same values - Map meta = parseMetaTags(sampleFile, true); + Map meta = parseMetaTags(sampleFile, true); assertEquals(description, getMeta(meta, MetaTagsParser.PARSE_META_PREFIX + "description")); @@ -200,7 +201,7 @@ public class TestMetaTagsParser { @Test public void testFilter() { // check that we get the same values - Map meta = parseMetaTags(sampleFile, false); + Map meta = parseMetaTags(sampleFile, false); assertEquals(description, getMeta(meta, MetaTagsParser.PARSE_META_PREFIX + "description")); @@ -208,7 +209,7 @@ public class TestMetaTagsParser { getMeta(meta, MetaTagsParser.PARSE_META_PREFIX + "keywords")); } - private String getMeta(Map meta, String name) { + private String getMeta(Map meta, String name) { ByteBuffer raw = meta.get(new Utf8(name)); return Bytes.toString(raw); } diff --git src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java index 52e0841..061d9cb 100644 --- src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java +++ src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java @@ -16,36 +16,15 @@ */ package org.apache.nutch.parse.tika; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; - import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.nutch.util.Bytes; import org.apache.html.dom.HTMLDocumentImpl; import org.apache.nutch.metadata.Nutch; -import org.apache.nutch.parse.HTMLMetaTags; -import org.apache.nutch.parse.ParseFilters; -import org.apache.nutch.parse.Outlink; -import org.apache.nutch.parse.OutlinkExtractor; -import org.apache.nutch.parse.Parse; -import org.apache.nutch.parse.ParseStatusCodes; -import org.apache.nutch.parse.ParseStatusUtils; -import org.apache.nutch.parse.ParseUtil; +import org.apache.nutch.parse.*; import org.apache.nutch.storage.ParseStatus; import org.apache.nutch.storage.WebPage; import org.apache.nutch.storage.WebPage.Field; +import org.apache.nutch.util.Bytes; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.TableUtil; @@ -53,8 +32,22 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.w3c.dom.DocumentFragment; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; + /** * Wrapper for Tika parsers. Mimics the HTMLParser but using the XHTML * representation returned by Tika as SAX events @@ -169,7 +162,7 @@ public class TikaParser implements org.apache.nutch.parse.Parser { if (tikaMDName.equalsIgnoreCase(TikaCoreProperties.TITLE.toString())) continue; // TODO what if multivalued? - page.putToMetadata(new Utf8(tikaMDName), ByteBuffer.wrap(Bytes.toBytes(tikamd + page.getMetadata().put(new Utf8(tikaMDName), ByteBuffer.wrap(Bytes.toBytes(tikamd .get(tikaMDName)))); } @@ -182,16 +175,16 @@ public class TikaParser implements org.apache.nutch.parse.Parser { ParseStatus status = ParseStatusUtils.STATUS_SUCCESS; if (metaTags.getRefresh()) { - status.setMinorCode(ParseStatusCodes.SUCCESS_REDIRECT); - status.addToArgs(new Utf8(metaTags.getRefreshHref().toString())); - status.addToArgs(new Utf8(Integer.toString(metaTags.getRefreshTime()))); + status.setMinorCode((int)ParseStatusCodes.SUCCESS_REDIRECT); + status.getArgs().add(new Utf8(metaTags.getRefreshHref().toString())); + status.getArgs().add(new Utf8(Integer.toString(metaTags.getRefreshTime()))); } Parse parse = new Parse(text, title, outlinks, status); parse = htmlParseFilters.filter(url, page, parse, metaTags, root); if (metaTags.getNoCache()) { // not okay to cache - page.putToMetadata(new Utf8(Nutch.CACHING_FORBIDDEN_KEY), ByteBuffer.wrap(Bytes + page.getMetadata().put(new Utf8(Nutch.CACHING_FORBIDDEN_KEY), ByteBuffer.wrap(Bytes .toBytes(cachingPolicy))); } @@ -241,7 +234,7 @@ public class TikaParser implements org.apache.nutch.parse.Parser { Configuration conf = NutchConfiguration.create(); // TikaParser parser = new TikaParser(); // parser.setConf(conf); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(url)); page.setContent(ByteBuffer.wrap(bytes)); MimeUtil mimeutil = new MimeUtil(conf); diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java index e54febc..130a90a 100644 --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java @@ -69,14 +69,14 @@ public class TestImageMetadata { parse = new ParseUtil(conf).parse(urlString, page); //assert width - ByteBuffer bbufW = page.getFromMetadata(new Utf8("width")); + ByteBuffer bbufW = page.getMetadata().get(new Utf8("width")); byte[] byteArrayW = new byte[bbufW.remaining()]; bbufW.get(byteArrayW); String width = new String(byteArrayW); assertEquals("121", width); //assert height - ByteBuffer bbufH = page.getFromMetadata(new Utf8("height")); + ByteBuffer bbufH = page.getMetadata().get(new Utf8("height")); byte[] byteArrayH = new byte[bbufH.remaining()]; bbufH.get(byteArrayH); String height = new String(byteArrayH); diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java index cf28353..db57414 100644 --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java @@ -17,16 +17,6 @@ package org.apache.nutch.parse.tika; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.parse.Parse; @@ -36,6 +26,16 @@ import org.apache.nutch.protocol.ProtocolException; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Before; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertTrue; /** * Unit tests for MSWordParser. @@ -72,7 +72,7 @@ public class TestMSWordParser { in.readFully(bytes); in.close(); Parse parse; - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8("file:"+urlString)); page.setContent(ByteBuffer.wrap(bytes)); // set the content type? diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java index 9ac94bc..eda90fd 100644 --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java @@ -17,16 +17,6 @@ package org.apache.nutch.parse.tika; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.ByteBuffer; - -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.parse.Parse; @@ -36,6 +26,12 @@ import org.apache.nutch.protocol.ProtocolException; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Test; + +import java.io.*; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertTrue; /** * Unit tests for OOParser. @@ -95,7 +91,7 @@ public class TestOOParser { in.readFully(bytes); in.close(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(urlString)); page.setContent(ByteBuffer.wrap(bytes)); String mtype = mimeutil.getMimeType(file); diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java index 68bcfa5..29aa333 100644 --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java @@ -17,15 +17,6 @@ package org.apache.nutch.parse.tika; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.parse.Parse; @@ -35,6 +26,15 @@ import org.apache.nutch.protocol.ProtocolException; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertTrue; /** * Unit tests for PdfParser. @@ -69,7 +69,7 @@ public class TestPdfParser { in.readFully(bytes); in.close(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(urlString)); page.setContent(ByteBuffer.wrap(bytes)); String mtype = mimeutil.getMimeType(file); diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java index 2015ad0..292fd4e 100644 --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java @@ -17,26 +17,26 @@ package org.apache.nutch.parse.tika; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.nutch.protocol.ProtocolException; - -import org.apache.nutch.parse.Parse; -import org.apache.nutch.parse.ParseUtil; -import org.apache.nutch.parse.ParseException; -import org.apache.nutch.parse.Outlink; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.parse.Outlink; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseException; +import org.apache.nutch.parse.ParseUtil; +import org.apache.nutch.protocol.ProtocolException; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; - import org.junit.Test; -import static org.junit.Assert.*; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Unit tests for the RSS Parser based on John Xing's TestPdfParser class. @@ -83,7 +83,7 @@ public class TestRSSParser { in.readFully(bytes); in.close(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(urlString)); page.setContent(ByteBuffer.wrap(bytes)); String mtype = mimeutil.getMimeType(file); diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java index cbf9ee1..01cf537 100644 --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java @@ -17,14 +17,6 @@ package org.apache.nutch.parse.tika; // JUnit imports -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.junit.Test; -import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -35,6 +27,15 @@ import org.apache.nutch.protocol.ProtocolException; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; /** * Unit tests for TestRTFParser. (Adapted from John Xing msword unit tests). @@ -67,7 +68,7 @@ public class TestRTFParser { in.readFully(bytes); in.close(); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8(urlString)); page.setContent(ByteBuffer.wrap(bytes)); String mtype = mimeutil.getMimeType(file); diff --git src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java index f7c523b..e70c134 100644 --- src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java +++ src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java @@ -16,29 +16,20 @@ */ package org.apache.nutch.protocol.file; -import java.net.URL; -import java.util.Collection; -import java.util.HashSet; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import crawlercommons.robots.BaseRobotRules; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; - import org.apache.nutch.net.protocols.Response; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.protocol.Protocol; -import org.apache.nutch.protocol.ProtocolOutput; -import org.apache.nutch.protocol.ProtocolStatusCodes; -import org.apache.nutch.protocol.ProtocolStatusUtils; -import org.apache.nutch.protocol.RobotRulesParser; +import org.apache.nutch.protocol.*; import org.apache.nutch.storage.ProtocolStatus; import org.apache.nutch.storage.WebPage; import org.apache.nutch.storage.WebPage.Field; import org.apache.nutch.util.NutchConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import crawlercommons.robots.BaseRobotRules; +import java.net.URL; +import java.util.Collection; +import java.util.HashSet; /** * This class is a protocol plugin used for file: scheme. @@ -183,7 +174,7 @@ public class File implements Protocol { if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength file.setMaxContentLength(maxContentLength); - Content content = file.getProtocolOutput(urlString, new WebPage()) + Content content = file.getProtocolOutput(urlString, WebPage.newBuilder().build()) .getContent(); System.out.println("Content-Type: " + content.getContentType()); diff --git src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java index 66dee16..beb005d 100644 --- src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java +++ src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java @@ -76,14 +76,14 @@ public class TestProtocolFile { public void setContentType(String testTextFile) throws ProtocolNotFound { String urlString = "file:" + sampleDir + fileSeparator + testTextFile; assertNotNull(urlString); - WebPage datum = new WebPage(); + WebPage datum = WebPage.newBuilder().build(); Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString); ProtocolOutput output = protocol.getProtocolOutput(urlString,datum); assertNotNull(output); assertEquals("Status code: [" + output.getStatus().getCode() + "], not equal to: [" + ProtocolStatusCodes.SUCCESS + "]: args: [" - + output.getStatus().getArgs() + "]", ProtocolStatusCodes.SUCCESS, output + + output.getStatus().getArgs() + "]", (Integer) ProtocolStatusCodes.SUCCESS, output .getStatus().getCode()); assertNotNull(output.getContent()); assertNotNull(output.getContent().getContentType()); diff --git src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java index 91a9460..64c6540 100644 --- src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java +++ src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java @@ -17,25 +17,20 @@ package org.apache.nutch.protocol.ftp; -import java.io.IOException; -import java.net.URL; -import java.util.Collection; -import java.util.HashSet; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import crawlercommons.robots.BaseRobotRules; import org.apache.commons.net.ftp.FTPFileEntryParser; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; -import org.apache.nutch.protocol.Content; -import org.apache.nutch.protocol.Protocol; -import org.apache.nutch.protocol.ProtocolOutput; -import org.apache.nutch.protocol.ProtocolStatusCodes; -import org.apache.nutch.protocol.ProtocolStatusUtils; +import org.apache.nutch.protocol.*; import org.apache.nutch.storage.ProtocolStatus; import org.apache.nutch.storage.WebPage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import crawlercommons.robots.BaseRobotRules; +import java.io.IOException; +import java.net.URL; +import java.util.Collection; +import java.util.HashSet; /** * This class is a protocol plugin used for ftp: scheme. @@ -238,7 +233,7 @@ public class Ftp implements Protocol { if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength ftp.setMaxContentLength(maxContentLength); - Content content = ftp.getProtocolOutput(urlString, new WebPage()) + Content content = ftp.getProtocolOutput(urlString, WebPage.newBuilder().build()) .getContent(); System.err.println("Content-Type: " + content.getContentType()); diff --git src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java index ea40a6d..9c5eb08 100644 --- src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java +++ src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java @@ -18,14 +18,6 @@ package org.apache.nutch.protocol.ftp; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.InetAddress; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.LinkedList; -import java.util.List; - import org.apache.avro.util.Utf8; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPFile; @@ -39,6 +31,14 @@ import org.apache.nutch.net.protocols.Response; import org.apache.nutch.protocol.Content; import org.apache.nutch.storage.WebPage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.List; + /************************************ * FtpResponse.java mimics ftp replies as http response. @@ -116,7 +116,7 @@ public class FtpResponse { if (addr != null && conf.getBoolean("store.ip.address", false) == true) { String ipString = addr.getHostAddress(); //get the ip address - page.putToMetadata(new Utf8("_ip_"), + page.getMetadata().put(new Utf8("_ip_"), ByteBuffer.wrap(ipString.getBytes())); } diff --git src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java index a5646d7..2c9cc0d 100644 --- src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java +++ src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java @@ -17,9 +17,8 @@ package org.apache.nutch.protocol.ftp; -import java.net.URL; - -import org.apache.commons.io.IOUtils; +import crawlercommons.robots.BaseRobotRules; +import crawlercommons.robots.SimpleRobotRules; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.protocol.Protocol; import org.apache.nutch.protocol.ProtocolOutput; @@ -29,8 +28,7 @@ import org.apache.nutch.storage.WebPage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import crawlercommons.robots.BaseRobotRules; -import crawlercommons.robots.SimpleRobotRules; +import java.net.URL; /** * This class is used for parsing robots for urls belonging to FTP protocol. @@ -75,7 +73,7 @@ public class FtpRobotRulesParser extends RobotRulesParser { try { String robotsUrl = new URL(url, "/robots.txt").toString(); - ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, new WebPage()); + ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, WebPage.newBuilder().build()); int statusCode = output.getStatus().getCode(); if (statusCode == ProtocolStatusCodes.SUCCESS) { diff --git src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java index 211b8dd..7896fef 100644 --- src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java +++ src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java @@ -17,17 +17,6 @@ package org.apache.nutch.protocol.http; // JDK imports -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PushbackInputStream; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.URL; -import java.nio.ByteBuffer; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -40,6 +29,12 @@ import org.apache.nutch.protocol.http.api.HttpBase; import org.apache.nutch.protocol.http.api.HttpException; import org.apache.nutch.storage.WebPage; +import java.io.*; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.nio.ByteBuffer; + /** An HTTP response. */ public class HttpResponse implements Response { @@ -97,7 +92,7 @@ public class HttpResponse implements Response { if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) { String ipString = sockAddr.getAddress().getHostAddress(); //get the ip address - page.putToMetadata(new Utf8("_ip_"), + page.getMetadata().put(new Utf8("_ip_"), ByteBuffer.wrap(ipString.getBytes())); } @@ -133,11 +128,11 @@ public class HttpResponse implements Response { reqStr.append("\r\n"); } - if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) { +// if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) { reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(page.getModifiedTime())); reqStr.append("\r\n"); - } +// } reqStr.append("\r\n"); byte[] reqBytes= reqStr.toString().getBytes(); @@ -177,7 +172,7 @@ public class HttpResponse implements Response { page.getHeaders().clear(); } for (String key : headers.names()) { - page.putToHeaders(new Utf8(key), new Utf8(headers.get(key))); + page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key))); } } finally { diff --git src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java index a5908be..82617d9 100644 --- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java +++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java @@ -176,7 +176,7 @@ public class HttpResponse implements Response { page.getHeaders().clear(); } for (String key : headers.names()) { - page.putToHeaders(new Utf8(key), new Utf8(headers.get(key))); + page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key))); } // Logger trace message diff --git src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java index 3a8d077..20c4bd8 100644 --- src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java +++ src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java @@ -17,22 +17,22 @@ package org.apache.nutch.protocol.httpclient; -import java.net.MalformedURLException; -import java.net.URL; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.storage.WebPage; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import org.mortbay.jetty.Server; import org.mortbay.jetty.nio.SelectChannelConnector; import org.mortbay.jetty.servlet.Context; import org.mortbay.jetty.servlet.ServletHolder; +import java.net.MalformedURLException; +import java.net.URL; + +import static org.junit.Assert.assertEquals; + /** * Test cases for protocol-httpclient. * @@ -210,7 +210,7 @@ public class TestProtocolHttpClient { private void fetchPage(String page, int expectedCode) throws Exception { URL url = new URL("http", "127.0.0.1", port, page); Response response = null; - response = http.getResponse(url, new WebPage(), true); + response = http.getResponse(url, WebPage.newBuilder().build(), true); int code = response.getCode(); assertEquals("HTTP Status Code for " + url, expectedCode, code); diff --git src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java index 29805b0..a51c837 100644 --- src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java +++ src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java @@ -17,17 +17,7 @@ package org.apache.nutch.scoring.opic; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.apache.avro.util.Utf8; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.NutchDocument; import org.apache.nutch.scoring.ScoreDatum; @@ -35,6 +25,16 @@ import org.apache.nutch.scoring.ScoringFilter; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; /** * This plugin implements a variant of an Online Page Importance Computation @@ -82,7 +82,7 @@ public class OPICScoringFilter implements ScoringFilter { public void injectedScore(String url, WebPage row) throws ScoringFilterException { float score = row.getScore(); - row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(score))); + row.getMetadata().put(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(score))); } /** Set to 0.0f (unknown value) - inlink contributions will bring it to @@ -90,7 +90,7 @@ public class OPICScoringFilter implements ScoringFilter { @Override public void initialScore(String url, WebPage row) throws ScoringFilterException { row.setScore(0.0f); - row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(0.0f))); + row.getMetadata().put(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(0.0f))); } /** Use {@link WebPage#getScore()}. */ @@ -108,12 +108,12 @@ public class OPICScoringFilter implements ScoringFilter { } float oldScore = row.getScore(); row.setScore(oldScore + adjust); - ByteBuffer cashRaw = row.getFromMetadata(CASH_KEY); + ByteBuffer cashRaw = row.getMetadata().get(CASH_KEY); float cash = 0.0f; if (cashRaw != null) { cash = Bytes.toFloat(cashRaw.array(), cashRaw.arrayOffset() + cashRaw.position()); } - row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(cash + adjust))); + row.getMetadata().put(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(cash + adjust))); } /** Get cash on hand, divide it by the number of outlinks and apply. */ @@ -121,7 +121,7 @@ public class OPICScoringFilter implements ScoringFilter { public void distributeScoreToOutlinks(String fromUrl, WebPage row, Collection scoreData, int allCount) { - ByteBuffer cashRaw = row.getFromMetadata(CASH_KEY); + ByteBuffer cashRaw = row.getMetadata().get(CASH_KEY); if (cashRaw == null) { return; } @@ -149,7 +149,7 @@ public class OPICScoringFilter implements ScoringFilter { } } // reset cash to zero - row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(0.0f))); + row.getMetadata().put(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(0.0f))); } /** Dampen the boost value by scorePower.*/ diff --git src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java index 9534f66..1357832 100644 --- src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java +++ src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java @@ -87,7 +87,7 @@ public class TestAdaptiveFetchSchedule extends TestCase { wp.setStatus(1); wp.setFetchInterval(interval); wp.setScore(1.0f); - wp.setFetchTime(0); + wp.setFetchTime(0L); return wp; } diff --git src/test/org/apache/nutch/crawl/TestGenerator.java src/test/org/apache/nutch/crawl/TestGenerator.java index f3fc019..a080d9f 100644 --- src/test/org/apache/nutch/crawl/TestGenerator.java +++ src/test/org/apache/nutch/crawl/TestGenerator.java @@ -16,13 +16,6 @@ */ package org.apache.nutch.crawl; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.WebPage; @@ -33,7 +26,15 @@ import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; + +import static org.junit.Assert.assertEquals; /** * Basic generator test. 1. Insert entries in webtable 2. Generates entries to @@ -294,10 +295,10 @@ public class TestGenerator extends AbstractNutchTest { */ private URLWebPage createURLWebPage(final String url, final int fetchInterval, final float score) { - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setFetchInterval(fetchInterval); page.setScore(score); - page.setStatus(CrawlStatus.STATUS_UNFETCHED); + page.setStatus((int)CrawlStatus.STATUS_UNFETCHED); return new URLWebPage(url, page); } diff --git src/test/org/apache/nutch/crawl/TestInjector.java src/test/org/apache/nutch/crawl/TestInjector.java index af4d85d..640f6e4 100644 --- src/test/org/apache/nutch/crawl/TestInjector.java +++ src/test/org/apache/nutch/crawl/TestInjector.java @@ -16,11 +16,6 @@ */ package org.apache.nutch.crawl; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - import org.apache.avro.util.Utf8; import org.apache.hadoop.fs.Path; import org.apache.nutch.storage.WebPage; @@ -30,7 +25,14 @@ import org.apache.nutch.util.CrawlTestUtil; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Basic injector test: 1. Creates a text file with urls 2. Injects them into @@ -110,8 +112,8 @@ public class TestInjector extends AbstractNutchTest { for (URLWebPage up : pages) { WebPage page = up.getDatum(); String representation = up.getUrl(); - representation += "\tnutch.score=" + (int)page.getScore(); - ByteBuffer bb = page.getFromMetadata(new Utf8("custom.attribute")); + representation += "\tnutch.score=" + page.getScore().intValue(); + ByteBuffer bb = page.getMetadata().get(new Utf8("custom.attribute")); if (bb != null) { representation += "\tcustom.attribute=" + Bytes.toString(bb); } diff --git src/test/org/apache/nutch/crawl/TestURLPartitioner.java src/test/org/apache/nutch/crawl/TestURLPartitioner.java index 2cfc364..be11d19 100644 --- src/test/org/apache/nutch/crawl/TestURLPartitioner.java +++ src/test/org/apache/nutch/crawl/TestURLPartitioner.java @@ -16,11 +16,6 @@ ******************************************************************************/ package org.apache.nutch.crawl; -import java.net.MalformedURLException; - -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IntWritable; import org.apache.nutch.crawl.GeneratorJob.SelectorEntry; @@ -30,6 +25,12 @@ import org.apache.nutch.fetcher.FetchEntry; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.TableUtil; +import org.junit.Test; + +import java.net.MalformedURLException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; /** * Tests {@link URLPartitioner} @@ -168,7 +169,7 @@ public class TestURLPartitioner { int partitionFromRef = refPartitioner.getPartition("http://www.example.org/", numReduceTasks); //init selector entry (score shouldn't matter) SelectorEntry selectorEntry = new SelectorEntry("http://www.example.org/", 1337); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); int partitionFromSig = sigPartitioner.getPartition(selectorEntry, page, numReduceTasks); assertEquals("partitions should be same", @@ -199,7 +200,7 @@ public class TestURLPartitioner { int partitionFromRef = refPartitioner.getPartition("http://www.example.org/", numReduceTasks); IntWritable intWritable = new IntWritable(1337); //doesn't matter - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); String key = TableUtil.reverseUrl("http://www.example.org/"); FetchEntry fetchEntry = new FetchEntry(conf, key, page); int partitionFromSig = sigPartitioner.getPartition(intWritable, fetchEntry, numReduceTasks); diff --git src/test/org/apache/nutch/indexer/TestIndexingFilters.java src/test/org/apache/nutch/indexer/TestIndexingFilters.java index c638952..429375c 100644 --- src/test/org/apache/nutch/indexer/TestIndexingFilters.java +++ src/test/org/apache/nutch/indexer/TestIndexingFilters.java @@ -16,14 +16,15 @@ */ package org.apache.nutch.indexer; -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class TestIndexingFilters { @@ -42,7 +43,7 @@ public class TestIndexingFilters { conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2); IndexingFilters filters = new IndexingFilters(conf); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setText(new Utf8("text")); page.setTitle(new Utf8("title")); filters.filter(new NutchDocument(),"http://www.example.com/",page); @@ -59,7 +60,7 @@ public class TestIndexingFilters { conf.addResource("crawl-tests.xml"); IndexingFilters filters = new IndexingFilters(conf); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setText(new Utf8("text")); page.setTitle(new Utf8("title")); NutchDocument doc = filters.filter(null,"http://www.example.com/",page); @@ -82,7 +83,7 @@ public class TestIndexingFilters { conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1); IndexingFilters filters1 = new IndexingFilters(conf); - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setText(new Utf8("text")); page.setTitle(new Utf8("title")); NutchDocument fdoc1 = filters1.filter(new NutchDocument(),"http://www.example.com/",page); diff --git src/test/org/apache/nutch/storage/TestGoraStorage.java src/test/org/apache/nutch/storage/TestGoraStorage.java index 2165dcd..eb9ff0d 100644 --- src/test/org/apache/nutch/storage/TestGoraStorage.java +++ src/test/org/apache/nutch/storage/TestGoraStorage.java @@ -16,16 +16,6 @@ ******************************************************************************/ package org.apache.nutch.storage; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - import org.apache.avro.util.Utf8; import org.apache.commons.io.IOUtils; import org.apache.gora.query.Result; @@ -34,12 +24,23 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.AbstractNutchTest; import org.apache.nutch.util.CrawlTestUtil; import org.hsqldb.Server; -import org.junit.Ignore; - import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * Tests basic Gora functionality by writing and reading webpages. @@ -71,7 +72,7 @@ public class TestGoraStorage extends AbstractNutchTest { private static void readWrite(String id, DataStore store) throws IOException, Exception { - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); int max = 1000; for (int i = 0; i < max; i++) { // store a page with title diff --git src/test/org/apache/nutch/util/CrawlTestUtil.java src/test/org/apache/nutch/util/CrawlTestUtil.java index ccf9e92..d2ff1e8 100644 --- src/test/org/apache/nutch/util/CrawlTestUtil.java +++ src/test/org/apache/nutch/util/CrawlTestUtil.java @@ -16,14 +16,9 @@ */ package org.apache.nutch.util; -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.gora.query.Query; +import org.apache.gora.query.Result; +import org.apache.gora.store.DataStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -31,14 +26,19 @@ import org.apache.hadoop.fs.Path; import org.apache.nutch.crawl.URLWebPage; import org.apache.nutch.storage.Mark; import org.apache.nutch.storage.WebPage; -import org.apache.gora.query.Query; -import org.apache.gora.query.Result; -import org.apache.gora.store.DataStore; import org.mortbay.jetty.Handler; import org.mortbay.jetty.Server; import org.mortbay.jetty.handler.DefaultHandler; import org.mortbay.jetty.handler.HandlerList; import org.mortbay.jetty.handler.ResourceHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; public class CrawlTestUtil { @@ -121,7 +121,7 @@ public class CrawlTestUtil { if (requiredMark != null && requiredMark.checkMark(page) == null) continue; - l.add(new URLWebPage(TableUtil.unreverseUrl(url), (WebPage)page.clone())); + l.add(new URLWebPage(TableUtil.unreverseUrl(url), WebPage.newBuilder(page).build())); } catch (Exception e) { e.printStackTrace(); } diff --git src/test/org/apache/nutch/util/TestEncodingDetector.java src/test/org/apache/nutch/util/TestEncodingDetector.java index d42f9ed..f4e8eb2 100644 --- src/test/org/apache/nutch/util/TestEncodingDetector.java +++ src/test/org/apache/nutch/util/TestEncodingDetector.java @@ -16,16 +16,16 @@ */ package org.apache.nutch.util; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; - -import org.junit.Test; -import static org.junit.Assert.*; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.storage.WebPage; +import org.junit.Test; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; public class TestEncodingDetector { private static Configuration conf = NutchConfiguration.create(); @@ -50,7 +50,7 @@ public class TestEncodingDetector { // Content content; String encoding; - WebPage page = new WebPage(); + WebPage page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8("http://www.example.com/")); page.setContentType(new Utf8("text/plain")); page.setContent(ByteBuffer.wrap(contentInOctets)); @@ -61,18 +61,18 @@ public class TestEncodingDetector { // no information is available, so it should return default encoding assertEquals("windows-1252", encoding.toLowerCase()); - page = new WebPage(); + page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8("http://www.example.com/")); page.setContentType(new Utf8("text/plain")); page.setContent(ByteBuffer.wrap(contentInOctets)); - page.putToHeaders(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/plain; charset=UTF-16")); + page.getHeaders().put(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/plain; charset=UTF-16")); detector = new EncodingDetector(conf); detector.autoDetectClues(page, true); encoding = detector.guessEncoding(page, "windows-1252"); assertEquals("utf-16", encoding.toLowerCase()); - page = new WebPage(); + page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8("http://www.example.com/")); page.setContentType(new Utf8("text/plain")); page.setContent(ByteBuffer.wrap(contentInOctets)); @@ -85,11 +85,11 @@ public class TestEncodingDetector { // enable autodetection conf.setInt(EncodingDetector.MIN_CONFIDENCE_KEY, 50); - page = new WebPage(); + page = WebPage.newBuilder().build(); page.setBaseUrl(new Utf8("http://www.example.com/")); page.setContentType(new Utf8("text/plain")); page.setContent(ByteBuffer.wrap(contentInOctets)); - page.putToMetadata(new Utf8(Response.CONTENT_TYPE), ByteBuffer.wrap("text/plain; charset=UTF-16".getBytes())); + page.getMetadata().put(new Utf8(Response.CONTENT_TYPE), ByteBuffer.wrap("text/plain; charset=UTF-16".getBytes())); detector = new EncodingDetector(conf); detector.autoDetectClues(page, true);