[-threads n] [-noParsing]";
@@ -551,15 +566,16 @@
System.err.println(usage);
return -1;
}
-
+
Path segment = new Path(args[0]);
int threads = getConf().getInt("fetcher.threads.fetch", 10);
boolean parsing = true;
- for (int i = 1; i < args.length; i++) { // parse command line
- if (args[i].equals("-threads")) { // found -threads option
- threads = Integer.parseInt(args[++i]);
- } else if (args[i].equals("-noParsing")) parsing = false;
+ for (int i = 1; i < args.length; i++) { // parse command line
+ if (args[i].equals("-threads")) { // found -threads option
+ threads = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-noParsing"))
+ parsing = false;
}
getConf().setInt("fetcher.threads.fetch", threads);
@@ -567,7 +583,7 @@
getConf().setBoolean("fetcher.parse", parsing);
}
try {
- fetch(segment, threads); // run the Fetcher
+ fetch(segment, threads); // run the Fetcher
return 0;
} catch (Exception e) {
LOG.error("OldFetcher: " + StringUtils.stringifyException(e));
Index: src/java/org/apache/nutch/fetcher/Fetcher.java
===================================================================
--- src/java/org/apache/nutch/fetcher/Fetcher.java (revision 1188252)
+++ src/java/org/apache/nutch/fetcher/Fetcher.java (working copy)
@@ -51,44 +51,46 @@
import org.apache.nutch.scoring.ScoringFilters;
import org.apache.nutch.util.*;
-
-/**
+/**
* A queue-based fetcher.
*
- * This fetcher uses a well-known model of one producer (a QueueFeeder)
- * and many consumers (FetcherThread-s).
+ *
+ * This fetcher uses a well-known model of one producer (a QueueFeeder) and many
+ * consumers (FetcherThread-s).
*
- *
QueueFeeder reads input fetchlists and
- * populates a set of FetchItemQueue-s, which hold FetchItem-s that
- * describe the items to be fetched. There are as many queues as there are unique
- * hosts, but at any given time the total number of fetch items in all queues
- * is less than a fixed number (currently set to a multiple of the number of
- * threads).
+ *
+ * QueueFeeder reads input fetchlists and populates a set of FetchItemQueue-s,
+ * which hold FetchItem-s that describe the items to be fetched. There are as
+ * many queues as there are unique hosts, but at any given time the total number
+ * of fetch items in all queues is less than a fixed number (currently set to a
+ * multiple of the number of threads).
*
- *
As items are consumed from the queues, the QueueFeeder continues to add new
+ *
+ * As items are consumed from the queues, the QueueFeeder continues to add new
* input items, so that their total count stays fixed (FetcherThread-s may also
* add new items to the queues e.g. as a results of redirection) - until all
* input items are exhausted, at which point the number of items in the queues
* begins to decrease. When this number reaches 0 fetcher will finish.
*
- *
This fetcher implementation handles per-host blocking itself, instead
- * of delegating this work to protocol-specific plugins.
- * Each per-host queue handles its own "politeness" settings, such as the
- * maximum number of concurrent requests and crawl delay between consecutive
- * requests - and also a list of requests in progress, and the time the last
- * request was finished. As FetcherThread-s ask for new items to be fetched,
- * queues may return eligible items or null if for "politeness" reasons this
- * host's queue is not yet ready.
+ *
+ * This fetcher implementation handles per-host blocking itself, instead of
+ * delegating this work to protocol-specific plugins. Each per-host queue
+ * handles its own "politeness" settings, such as the maximum number of
+ * concurrent requests and crawl delay between consecutive requests - and also a
+ * list of requests in progress, and the time the last request was finished. As
+ * FetcherThread-s ask for new items to be fetched, queues may return eligible
+ * items or null if for "politeness" reasons this host's queue is not yet ready.
*
- *
If there are still unfetched items in the queues, but none of the items
- * are ready, FetcherThread-s will spin-wait until either some items become
+ *
+ * If there are still unfetched items in the queues, but none of the items are
+ * ready, FetcherThread-s will spin-wait until either some items become
* available, or a timeout is reached (at which point the Fetcher will abort,
* assuming the task is hung).
*
* @author Andrzej Bialecki
*/
public class Fetcher extends Configured implements Tool,
- MapRunnable {
+ MapRunnable {
public static final int PERM_REFRESH_TIME = 5;
@@ -97,17 +99,17 @@
public static final String PROTOCOL_REDIR = "protocol";
public static final Logger LOG = LoggerFactory.getLogger(Fetcher.class);
-
- public static class InputFormat extends SequenceFileInputFormat {
+
+ public static class InputFormat extends
+ SequenceFileInputFormat {
/** Don't split inputs, to keep things polite. */
- public InputSplit[] getSplits(JobConf job, int nSplits)
- throws IOException {
+ public InputSplit[] getSplits(JobConf job, int nSplits) throws IOException {
FileStatus[] files = listStatus(job);
FileSplit[] splits = new FileSplit[files.length];
for (int i = 0; i < files.length; i++) {
FileStatus cur = files[i];
- splits[i] = new FileSplit(cur.getPath(), 0,
- cur.getLen(), (String[])null);
+ splits[i] = new FileSplit(cur.getPath(), 0, cur.getLen(),
+ (String[]) null);
}
return splits;
}
@@ -115,7 +117,7 @@
private OutputCollector output;
private Reporter reporter;
-
+
private String segmentName;
private AtomicInteger activeThreads = new AtomicInteger(0);
private AtomicInteger spinWaiting = new AtomicInteger(0);
@@ -123,36 +125,37 @@
private long start = System.currentTimeMillis(); // start time of fetcher run
private AtomicLong lastRequestStart = new AtomicLong(start);
- private AtomicLong bytes = new AtomicLong(0); // total bytes fetched
- private AtomicInteger pages = new AtomicInteger(0); // total pages fetched
+ private AtomicLong bytes = new AtomicLong(0); // total bytes fetched
+ private AtomicInteger pages = new AtomicInteger(0); // total pages fetched
private AtomicInteger errors = new AtomicInteger(0); // total pages errored
private boolean storingContent;
private boolean parsing;
FetchItemQueues fetchQueues;
QueueFeeder feeder;
-
+
/**
* This class described the item to be fetched.
*/
- private static class FetchItem {
+ private static class FetchItem {
String queueID;
Text url;
URL u;
CrawlDatum datum;
-
+
public FetchItem(Text url, URL u, CrawlDatum datum, String queueID) {
this.url = url;
this.u = u;
this.datum = datum;
this.queueID = queueID;
}
-
- /** Create an item. Queue id will be created based on queueMode
- * argument, either as a protocol + hostname pair, protocol + IP
- * address pair or protocol+domain pair.
+
+ /**
+ * Create an item. Queue id will be created based on queueMode
+ * argument, either as a protocol + hostname pair, protocol + IP address
+ * pair or protocol+domain pair.
*/
- public static FetchItem create(Text url, CrawlDatum datum, String queueMode) {
+ public static FetchItem create(Text url, CrawlDatum datum, String queueMode) {
String queueID;
URL u = null;
try {
@@ -172,19 +175,18 @@
LOG.warn("Unable to resolve: " + u.getHost() + ", skipping.");
return null;
}
- }
- else if (FetchItemQueues.QUEUE_MODE_DOMAIN.equalsIgnoreCase(queueMode)){
+ } else if (FetchItemQueues.QUEUE_MODE_DOMAIN.equalsIgnoreCase(queueMode)) {
key = URLUtil.getDomainName(u);
if (key == null) {
- LOG.warn("Unknown domain for url: " + url + ", using URL string as key");
- key=u.toExternalForm();
+ LOG.warn("Unknown domain for url: " + url
+ + ", using URL string as key");
+ key = u.toExternalForm();
}
- }
- else {
+ } else {
key = u.getHost();
if (key == null) {
LOG.warn("Unknown host for url: " + url + ", using URL string as key");
- key=u.toExternalForm();
+ key = u.toExternalForm();
}
}
queueID = proto + "://" + key.toLowerCase();
@@ -202,28 +204,31 @@
public Text getUrl() {
return url;
}
-
+
public URL getURL2() {
return u;
}
}
-
+
/**
- * This class handles FetchItems which come from the same host ID (be it
- * a proto/hostname or proto/IP pair). It also keeps track of requests in
+ * This class handles FetchItems which come from the same host ID (be it a
+ * proto/hostname or proto/IP pair). It also keeps track of requests in
* progress and elapsed time between requests.
*/
private static class FetchItemQueue {
- List queue = Collections.synchronizedList(new LinkedList());
- Set inProgress = Collections.synchronizedSet(new HashSet());
+ List queue = Collections
+ .synchronizedList(new LinkedList());
+ Set inProgress = Collections
+ .synchronizedSet(new HashSet());
AtomicLong nextFetchTime = new AtomicLong();
AtomicInteger exceptionCounter = new AtomicInteger();
long crawlDelay;
long minCrawlDelay;
int maxThreads;
Configuration conf;
-
- public FetchItemQueue(Configuration conf, int maxThreads, long crawlDelay, long minCrawlDelay) {
+
+ public FetchItemQueue(Configuration conf, int maxThreads, long crawlDelay,
+ long minCrawlDelay) {
this.conf = conf;
this.maxThreads = maxThreads;
this.crawlDelay = crawlDelay;
@@ -231,57 +236,64 @@
// ready to start
setEndTime(System.currentTimeMillis() - crawlDelay);
}
-
+
public synchronized int emptyQueue() {
int presize = queue.size();
queue.clear();
return presize;
}
-
+
public int getQueueSize() {
return queue.size();
}
-
+
public int getInProgressSize() {
return inProgress.size();
}
-
+
public int incrementExceptionCounter() {
return exceptionCounter.incrementAndGet();
}
-
+
public void finishFetchItem(FetchItem it, boolean asap) {
if (it != null) {
inProgress.remove(it);
setEndTime(System.currentTimeMillis(), asap);
}
}
-
+
public void addFetchItem(FetchItem it) {
- if (it == null) return;
+ if (it == null)
+ return;
queue.add(it);
}
-
+
public void addInProgressFetchItem(FetchItem it) {
- if (it == null) return;
+ if (it == null)
+ return;
inProgress.add(it);
}
-
+
public FetchItem getFetchItem() {
- if (inProgress.size() >= maxThreads) return null;
+ if (inProgress.size() >= maxThreads)
+ return null;
long now = System.currentTimeMillis();
- if (nextFetchTime.get() > now) return null;
+ if (nextFetchTime.get() > now)
+ return null;
FetchItem it = null;
- if (queue.size() == 0) return null;
+ if (queue.size() == 0)
+ return null;
try {
it = queue.remove(0);
inProgress.add(it);
} catch (Exception e) {
- LOG.error("Cannot remove FetchItem from queue or cannot add it to inProgress queue", e);
+ LOG.error(
+ "Cannot remove FetchItem from queue or cannot add it to inProgress queue",
+ e);
}
return it;
}
-
+
public synchronized void dump() {
LOG.info(" maxThreads = " + maxThreads);
LOG.info(" inProgress = " + inProgress.size());
@@ -294,19 +306,20 @@
LOG.info(" " + i + ". " + it.url);
}
}
-
+
private void setEndTime(long endTime) {
setEndTime(endTime, false);
}
-
+
private void setEndTime(long endTime, boolean asap) {
if (!asap)
- nextFetchTime.set(endTime + (maxThreads > 1 ? minCrawlDelay : crawlDelay));
+ nextFetchTime.set(endTime
+ + (maxThreads > 1 ? minCrawlDelay : crawlDelay));
else
nextFetchTime.set(endTime);
}
}
-
+
/**
* Convenience class - a collection of queues that keeps track of the total
* number of items, and provides items eligible for fetching from any queue.
@@ -320,55 +333,60 @@
long minCrawlDelay;
long timelimit = -1;
int maxExceptionsPerQueue = -1;
- Configuration conf;
+ Configuration conf;
public static final String QUEUE_MODE_HOST = "byHost";
public static final String QUEUE_MODE_DOMAIN = "byDomain";
public static final String QUEUE_MODE_IP = "byIP";
-
+
String queueMode;
-
+
public FetchItemQueues(Configuration conf) {
this.conf = conf;
this.maxThreads = conf.getInt("fetcher.threads.per.queue", 1);
queueMode = conf.get("fetcher.queue.mode", QUEUE_MODE_HOST);
// check that the mode is known
- if (!queueMode.equals(QUEUE_MODE_IP) && !queueMode.equals(QUEUE_MODE_DOMAIN)
+ if (!queueMode.equals(QUEUE_MODE_IP)
+ && !queueMode.equals(QUEUE_MODE_DOMAIN)
&& !queueMode.equals(QUEUE_MODE_HOST)) {
- LOG.error("Unknown partition mode : " + queueMode + " - forcing to byHost");
+ LOG.error("Unknown partition mode : " + queueMode
+ + " - forcing to byHost");
queueMode = QUEUE_MODE_HOST;
}
- LOG.info("Using queue mode : "+queueMode);
-
+ LOG.info("Using queue mode : " + queueMode);
+
this.crawlDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
- this.minCrawlDelay = (long) (conf.getFloat("fetcher.server.min.delay", 0.0f) * 1000);
+ this.minCrawlDelay = (long) (conf.getFloat("fetcher.server.min.delay",
+ 0.0f) * 1000);
this.timelimit = conf.getLong("fetcher.timelimit", -1);
- this.maxExceptionsPerQueue = conf.getInt("fetcher.max.exceptions.per.queue", -1);
+ this.maxExceptionsPerQueue = conf.getInt(
+ "fetcher.max.exceptions.per.queue", -1);
}
-
+
public int getTotalSize() {
return totalSize.get();
}
-
+
public int getQueueCount() {
return queues.size();
}
-
+
public void addFetchItem(Text url, CrawlDatum datum) {
FetchItem it = FetchItem.create(url, datum, queueMode);
- if (it != null) addFetchItem(it);
+ if (it != null)
+ addFetchItem(it);
}
-
+
public synchronized void addFetchItem(FetchItem it) {
FetchItemQueue fiq = getFetchItemQueue(it.queueID);
fiq.addFetchItem(it);
totalSize.incrementAndGet();
}
-
+
public void finishFetchItem(FetchItem it) {
finishFetchItem(it, false);
}
-
+
public void finishFetchItem(FetchItem it, boolean asap) {
FetchItemQueue fiq = queues.get(it.queueID);
if (fiq == null) {
@@ -377,7 +395,7 @@
}
fiq.finishFetchItem(it, asap);
}
-
+
public synchronized FetchItemQueue getFetchItemQueue(String id) {
FetchItemQueue fiq = queues.get(id);
if (fiq == null) {
@@ -387,10 +405,10 @@
}
return fiq;
}
-
+
public synchronized FetchItem getFetchItem() {
- Iterator> it =
- queues.entrySet().iterator();
+ Iterator> it = queues.entrySet()
+ .iterator();
while (it.hasNext()) {
FetchItemQueue fiq = it.next().getValue();
// reap empty queues
@@ -406,7 +424,7 @@
}
return null;
}
-
+
// called only once the feeder has stopped
public synchronized int checkTimelimit() {
int count = 0;
@@ -418,7 +436,8 @@
// there might also be a case where totalsize !=0 but number of queues
// == 0
// in which case we simply force it to 0 to avoid blocking
- if (totalSize.get() != 0 && queues.size() == 0) totalSize.set(0);
+ if (totalSize.get() != 0 && queues.size() == 0)
+ totalSize.set(0);
}
return count;
}
@@ -429,7 +448,8 @@
for (String id : queues.keySet()) {
FetchItemQueue fiq = queues.get(id);
- if (fiq.getQueueSize() == 0) continue;
+ if (fiq.getQueueSize() == 0)
+ continue;
LOG.info("* queue: " + id + " >> dropping! ");
int deleted = fiq.emptyQueue();
for (int i = 0; i < deleted; i++) {
@@ -440,11 +460,11 @@
return count;
}
-
+
/**
* Increment the exception counter of a queue in case of an exception e.g.
* timeout; when higher than a given threshold simply empty the queue.
- *
+ *
* @param queueid
* @return number of purged items
*/
@@ -457,7 +477,7 @@
return 0;
}
int excCount = fiq.incrementExceptionCounter();
- if (maxExceptionsPerQueue!= -1 && excCount >= maxExceptionsPerQueue) {
+ if (maxExceptionsPerQueue != -1 && excCount >= maxExceptionsPerQueue) {
// too many exceptions for items in this queue - purge it
int deleted = fiq.emptyQueue();
LOG.info("* queue: " + queueid + " >> removed " + deleted
@@ -470,20 +490,20 @@
return 0;
}
-
public synchronized void dump() {
for (String id : queues.keySet()) {
FetchItemQueue fiq = queues.get(id);
- if (fiq.getQueueSize() == 0) continue;
+ if (fiq.getQueueSize() == 0)
+ continue;
LOG.info("* queue: " + id);
fiq.dump();
}
}
}
-
+
/**
- * This class feeds the queues with input items, and re-fills them as
- * items are consumed by FetcherThread-s.
+ * This class feeds the queues with input items, and re-fills them as items
+ * are consumed by FetcherThread-s.
*/
private static class QueueFeeder extends Thread {
private RecordReader reader;
@@ -499,7 +519,7 @@
this.setDaemon(true);
this.setName("QueueFeeder");
}
-
+
public void setTimeLimit(long tl) {
timelimit = tl;
}
@@ -528,7 +548,9 @@
// queues are full - spin-wait until they have some free space
try {
Thread.sleep(1000);
- } catch (Exception e) {};
+ } catch (Exception e) {
+ }
+ ;
continue;
} else {
LOG.debug("-feeding " + feed + " input urls ...");
@@ -549,11 +571,11 @@
}
}
}
- LOG.info("QueueFeeder finished: total " + cnt + " records + hit by time limit :"
- + timelimitcount);
+ LOG.info("QueueFeeder finished: total " + cnt
+ + " records + hit by time limit :" + timelimitcount);
}
}
-
+
/**
* This class picks items from queues and fetches the pages.
*/
@@ -573,8 +595,8 @@
private boolean ignoreExternalLinks;
public FetcherThread(Configuration conf) {
- this.setDaemon(true); // don't hang JVM on exit
- this.setName("FetcherThread"); // use an informative name
+ this.setDaemon(true); // don't hang JVM on exit
+ this.setName("FetcherThread"); // use an informative name
this.conf = conf;
this.urlFilters = new URLFilters(conf);
this.scfilters = new ScoringFilters(conf);
@@ -582,25 +604,28 @@
this.protocolFactory = new ProtocolFactory(conf);
this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER);
this.maxCrawlDelay = conf.getInt("fetcher.max.crawl.delay", 30) * 1000;
- queueMode = conf.get("fetcher.queue.mode", FetchItemQueues.QUEUE_MODE_HOST);
+ queueMode = conf.get("fetcher.queue.mode",
+ FetchItemQueues.QUEUE_MODE_HOST);
// check that the mode is known
- if (!queueMode.equals(FetchItemQueues.QUEUE_MODE_IP) && !queueMode.equals(FetchItemQueues.QUEUE_MODE_DOMAIN)
+ if (!queueMode.equals(FetchItemQueues.QUEUE_MODE_IP)
+ && !queueMode.equals(FetchItemQueues.QUEUE_MODE_DOMAIN)
&& !queueMode.equals(FetchItemQueues.QUEUE_MODE_HOST)) {
- LOG.error("Unknown partition mode : " + queueMode + " - forcing to byHost");
+ LOG.error("Unknown partition mode : " + queueMode
+ + " - forcing to byHost");
queueMode = FetchItemQueues.QUEUE_MODE_HOST;
}
- LOG.info("Using queue mode : "+queueMode);
+ LOG.info("Using queue mode : " + queueMode);
this.maxRedirect = conf.getInt("http.redirect.max", 3);
- this.ignoreExternalLinks =
- conf.getBoolean("db.ignore.external.links", false);
+ this.ignoreExternalLinks = conf.getBoolean("db.ignore.external.links",
+ false);
}
public void run() {
activeThreads.incrementAndGet(); // count threads
-
+
FetchItem fit = null;
try {
-
+
while (true) {
fit = fetchQueues.getFetchItem();
if (fit == null) {
@@ -610,8 +635,9 @@
spinWaiting.incrementAndGet();
try {
Thread.sleep(500);
- } catch (Exception e) {}
- spinWaiting.decrementAndGet();
+ } catch (Exception e) {
+ }
+ spinWaiting.decrementAndGet();
continue;
} else {
// all done, finish this thread
@@ -619,15 +645,17 @@
}
}
lastRequestStart.set(System.currentTimeMillis());
- Text reprUrlWritable =
- (Text) fit.datum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY);
+ Text reprUrlWritable = (Text) fit.datum.getMetaData().get(
+ Nutch.WRITABLE_REPR_URL_KEY);
if (reprUrlWritable == null) {
reprUrl = fit.url.toString();
} else {
reprUrl = reprUrlWritable.toString();
}
try {
- if (LOG.isInfoEnabled()) { LOG.info("fetching " + fit.url); }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("fetching " + fit.url);
+ }
// fetch the page
redirecting = false;
@@ -637,7 +665,8 @@
LOG.debug("redirectCount=" + redirectCount);
}
redirecting = false;
- Protocol protocol = this.protocolFactory.getProtocol(fit.url.toString());
+ Protocol protocol = this.protocolFactory.getProtocol(fit.url
+ .toString());
RobotRules rules = protocol.getRobotRules(fit.url, fit.datum);
if (!rules.isAllowed(fit.u)) {
// unblock
@@ -645,7 +674,9 @@
if (LOG.isDebugEnabled()) {
LOG.debug("Denied by robots.txt: " + fit.url);
}
- output(fit.url, fit.datum, null, ProtocolStatus.STATUS_ROBOTS_DENIED, CrawlDatum.STATUS_FETCH_GONE);
+ output(fit.url, fit.datum, null,
+ ProtocolStatus.STATUS_ROBOTS_DENIED,
+ CrawlDatum.STATUS_FETCH_GONE);
reporter.incrCounter("FetcherStatus", "robots_denied", 1);
continue;
}
@@ -653,16 +684,22 @@
if (rules.getCrawlDelay() > maxCrawlDelay) {
// unblock
fetchQueues.finishFetchItem(fit, true);
- LOG.debug("Crawl-Delay for " + fit.url + " too long (" + rules.getCrawlDelay() + "), skipping");
- output(fit.url, fit.datum, null, ProtocolStatus.STATUS_ROBOTS_DENIED, CrawlDatum.STATUS_FETCH_GONE);
- reporter.incrCounter("FetcherStatus", "robots_denied_maxcrawldelay", 1);
+ LOG.debug("Crawl-Delay for " + fit.url + " too long ("
+ + rules.getCrawlDelay() + "), skipping");
+ output(fit.url, fit.datum, null,
+ ProtocolStatus.STATUS_ROBOTS_DENIED,
+ CrawlDatum.STATUS_FETCH_GONE);
+ reporter.incrCounter("FetcherStatus",
+ "robots_denied_maxcrawldelay", 1);
continue;
} else {
- FetchItemQueue fiq = fetchQueues.getFetchItemQueue(fit.queueID);
+ FetchItemQueue fiq = fetchQueues
+ .getFetchItemQueue(fit.queueID);
fiq.crawlDelay = rules.getCrawlDelay();
}
}
- ProtocolOutput output = protocol.getProtocolOutput(fit.url, fit.datum);
+ ProtocolOutput output = protocol.getProtocolOutput(fit.url,
+ fit.datum);
ProtocolStatus status = output.getStatus();
Content content = output.getContent();
ParseStatus pstatus = null;
@@ -672,28 +709,28 @@
String urlString = fit.url.toString();
reporter.incrCounter("FetcherStatus", status.getName(), 1);
-
- switch(status.getCode()) {
-
+
+ switch (status.getCode()) {
+
case ProtocolStatus.WOULDBLOCK:
// retry ?
fetchQueues.addFetchItem(fit);
break;
- case ProtocolStatus.SUCCESS: // got a page
- pstatus = output(fit.url, fit.datum, content, status, CrawlDatum.STATUS_FETCH_SUCCESS);
+ case ProtocolStatus.SUCCESS: // got a page
+ pstatus = output(fit.url, fit.datum, content, status,
+ CrawlDatum.STATUS_FETCH_SUCCESS);
updateStatus(content.getContent().length);
- if (pstatus != null && pstatus.isSuccess() &&
- pstatus.getMinorCode() == ParseStatus.SUCCESS_REDIRECT) {
+ if (pstatus != null && pstatus.isSuccess()
+ && pstatus.getMinorCode() == ParseStatus.SUCCESS_REDIRECT) {
String newUrl = pstatus.getMessage();
int refreshTime = Integer.valueOf(pstatus.getArgs()[1]);
- Text redirUrl =
- handleRedirect(fit.url, fit.datum,
- urlString, newUrl,
- refreshTime < Fetcher.PERM_REFRESH_TIME,
- Fetcher.CONTENT_REDIR);
+ Text redirUrl = handleRedirect(fit.url, fit.datum, urlString,
+ newUrl, refreshTime < Fetcher.PERM_REFRESH_TIME,
+ Fetcher.CONTENT_REDIR);
if (redirUrl != null) {
- CrawlDatum newDatum = new CrawlDatum(CrawlDatum.STATUS_DB_UNFETCHED,
+ CrawlDatum newDatum = new CrawlDatum(
+ CrawlDatum.STATUS_DB_UNFETCHED,
fit.datum.getFetchInterval(), fit.datum.getScore());
// transfer existing metadata to the redir
newDatum.getMetaData().putAll(fit.datum.getMetaData());
@@ -704,19 +741,20 @@
}
fit = FetchItem.create(redirUrl, newDatum, queueMode);
if (fit != null) {
- FetchItemQueue fiq =
- fetchQueues.getFetchItemQueue(fit.queueID);
+ FetchItemQueue fiq = fetchQueues
+ .getFetchItemQueue(fit.queueID);
fiq.addInProgressFetchItem(fit);
} else {
// stop redirecting
redirecting = false;
- reporter.incrCounter("FetcherStatus", "FetchItem.notCreated.redirect", 1);
+ reporter.incrCounter("FetcherStatus",
+ "FetchItem.notCreated.redirect", 1);
}
}
}
break;
- case ProtocolStatus.MOVED: // redirect
+ case ProtocolStatus.MOVED: // redirect
case ProtocolStatus.TEMP_MOVED:
int code;
boolean temp;
@@ -729,12 +767,11 @@
}
output(fit.url, fit.datum, content, status, code);
String newUrl = status.getMessage();
- Text redirUrl =
- handleRedirect(fit.url, fit.datum,
- urlString, newUrl, temp,
- Fetcher.PROTOCOL_REDIR);
+ Text redirUrl = handleRedirect(fit.url, fit.datum, urlString,
+ newUrl, temp, Fetcher.PROTOCOL_REDIR);
if (redirUrl != null) {
- CrawlDatum newDatum = new CrawlDatum(CrawlDatum.STATUS_DB_UNFETCHED,
+ CrawlDatum newDatum = new CrawlDatum(
+ CrawlDatum.STATUS_DB_UNFETCHED,
fit.datum.getFetchInterval(), fit.datum.getScore());
// transfer existing metadata
newDatum.getMetaData().putAll(fit.datum.getMetaData());
@@ -745,13 +782,14 @@
}
fit = FetchItem.create(redirUrl, newDatum, queueMode);
if (fit != null) {
- FetchItemQueue fiq =
- fetchQueues.getFetchItemQueue(fit.queueID);
+ FetchItemQueue fiq = fetchQueues
+ .getFetchItemQueue(fit.queueID);
fiq.addInProgressFetchItem(fit);
} else {
// stop redirecting
redirecting = false;
- reporter.incrCounter("FetcherStatus", "FetchItem.notCreated.redirect", 1);
+ reporter.incrCounter("FetcherStatus",
+ "FetchItem.notCreated.redirect", 1);
}
} else {
// stop redirecting
@@ -761,31 +799,37 @@
case ProtocolStatus.EXCEPTION:
logError(fit.url, status.getMessage());
- int killedURLs = fetchQueues.checkExceptionThreshold(fit.getQueueID());
- if (killedURLs!=0)
- reporter.incrCounter("FetcherStatus", "AboveExceptionThresholdInQueue", killedURLs);
+ int killedURLs = fetchQueues.checkExceptionThreshold(fit
+ .getQueueID());
+ if (killedURLs != 0)
+ reporter.incrCounter("FetcherStatus",
+ "AboveExceptionThresholdInQueue", killedURLs);
/* FALLTHROUGH */
- case ProtocolStatus.RETRY: // retry
+ case ProtocolStatus.RETRY: // retry
case ProtocolStatus.BLOCKED:
- output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_FETCH_RETRY);
+ output(fit.url, fit.datum, null, status,
+ CrawlDatum.STATUS_FETCH_RETRY);
break;
-
- case ProtocolStatus.GONE: // gone
+
+ case ProtocolStatus.GONE: // gone
case ProtocolStatus.NOTFOUND:
case ProtocolStatus.ACCESS_DENIED:
case ProtocolStatus.ROBOTS_DENIED:
- output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_FETCH_GONE);
+ output(fit.url, fit.datum, null, status,
+ CrawlDatum.STATUS_FETCH_GONE);
break;
case ProtocolStatus.NOTMODIFIED:
- output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_FETCH_NOTMODIFIED);
+ output(fit.url, fit.datum, null, status,
+ CrawlDatum.STATUS_FETCH_NOTMODIFIED);
break;
default:
if (LOG.isWarnEnabled()) {
LOG.warn("Unknown ProtocolStatus: " + status.getCode());
}
- output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_FETCH_RETRY);
+ output(fit.url, fit.datum, null, status,
+ CrawlDatum.STATUS_FETCH_RETRY);
}
if (redirecting && redirectCount > maxRedirect) {
@@ -793,53 +837,58 @@
if (LOG.isInfoEnabled()) {
LOG.info(" - redirect count exceeded " + fit.url);
}
- output(fit.url, fit.datum, null, ProtocolStatus.STATUS_REDIR_EXCEEDED, CrawlDatum.STATUS_FETCH_GONE);
+ output(fit.url, fit.datum, null,
+ ProtocolStatus.STATUS_REDIR_EXCEEDED,
+ CrawlDatum.STATUS_FETCH_GONE);
}
} while (redirecting && (redirectCount <= maxRedirect));
-
- } catch (Throwable t) { // unexpected exception
+
+ } catch (Throwable t) { // unexpected exception
// unblock
fetchQueues.finishFetchItem(fit);
logError(fit.url, t.toString());
- output(fit.url, fit.datum, null, ProtocolStatus.STATUS_FAILED, CrawlDatum.STATUS_FETCH_RETRY);
+ output(fit.url, fit.datum, null, ProtocolStatus.STATUS_FAILED,
+ CrawlDatum.STATUS_FETCH_RETRY);
}
}
} catch (Throwable e) {
if (LOG.isErrorEnabled()) {
e.printStackTrace(LogUtil.getErrorStream(LOG));
- LOG.error("fetcher caught:"+e.toString());
+ LOG.error("fetcher caught:" + e.toString());
}
} finally {
- if (fit != null) fetchQueues.finishFetchItem(fit);
+ if (fit != null)
+ fetchQueues.finishFetchItem(fit);
activeThreads.decrementAndGet(); // count threads
- LOG.info("-finishing thread " + getName() + ", activeThreads=" + activeThreads);
+ LOG.info("-finishing thread " + getName() + ", activeThreads="
+ + activeThreads);
}
}
- private Text handleRedirect(Text url, CrawlDatum datum,
- String urlString, String newUrl,
- boolean temp, String redirType)
- throws MalformedURLException, URLFilterException {
+ private Text handleRedirect(Text url, CrawlDatum datum, String urlString,
+ String newUrl, boolean temp, String redirType)
+ throws MalformedURLException, URLFilterException {
newUrl = normalizers.normalize(newUrl, URLNormalizers.SCOPE_FETCHER);
newUrl = urlFilters.filter(newUrl);
-
+
if (ignoreExternalLinks) {
try {
String origHost = new URL(urlString).getHost().toLowerCase();
String newHost = new URL(newUrl).getHost().toLowerCase();
if (!origHost.equals(newHost)) {
if (LOG.isDebugEnabled()) {
- LOG.debug(" - ignoring redirect " + redirType + " from " +
- urlString + " to " + newUrl +
- " because external links are ignored");
+ LOG.debug(" - ignoring redirect " + redirType + " from "
+ + urlString + " to " + newUrl
+ + " because external links are ignored");
}
return null;
}
- } catch (MalformedURLException e) { }
+ } catch (MalformedURLException e) {
+ }
}
-
+
if (newUrl != null && !newUrl.equals(urlString)) {
reprUrl = URLUtil.chooseRepr(reprUrl, newUrl, temp);
url = new Text(newUrl);
@@ -847,14 +896,14 @@
redirecting = true;
redirectCount++;
if (LOG.isDebugEnabled()) {
- LOG.debug(" - " + redirType + " redirect to " +
- url + " (fetching now)");
+ LOG.debug(" - " + redirType + " redirect to " + url
+ + " (fetching now)");
}
return url;
} else {
CrawlDatum newDatum = new CrawlDatum(CrawlDatum.STATUS_LINKED,
- datum.getFetchInterval(),datum.getScore());
- // transfer existing metadata
+ datum.getFetchInterval(), datum.getScore());
+ // transfer existing metadata
newDatum.getMetaData().putAll(datum.getMetaData());
try {
scfilters.initialScore(url, newDatum);
@@ -867,15 +916,15 @@
}
output(url, newDatum, null, null, CrawlDatum.STATUS_LINKED);
if (LOG.isDebugEnabled()) {
- LOG.debug(" - " + redirType + " redirect to " +
- url + " (fetching later)");
+ LOG.debug(" - " + redirType + " redirect to " + url
+ + " (fetching later)");
}
return null;
}
} else {
if (LOG.isDebugEnabled()) {
- LOG.debug(" - " + redirType + " redirect skipped: " +
- (newUrl != null ? "to same url" : "filtered"));
+ LOG.debug(" - " + redirType + " redirect skipped: "
+ + (newUrl != null ? "to same url" : "filtered"));
}
return null;
}
@@ -888,12 +937,13 @@
errors.incrementAndGet();
}
- private ParseStatus output(Text key, CrawlDatum datum,
- Content content, ProtocolStatus pstatus, int status) {
+ private ParseStatus output(Text key, CrawlDatum datum, Content content,
+ ProtocolStatus pstatus, int status) {
datum.setStatus(status);
datum.setFetchTime(System.currentTimeMillis());
- if (pstatus != null) datum.getMetaData().put(Nutch.WRITABLE_PROTO_STATUS_KEY, pstatus);
+ if (pstatus != null)
+ datum.getMetaData().put(Nutch.WRITABLE_PROTO_STATUS_KEY, pstatus);
ParseResult parseResult = null;
if (content != null) {
@@ -909,27 +959,31 @@
LOG.warn("Couldn't pass score, url " + key + " (" + e + ")");
}
}
- /* Note: Fetcher will only follow meta-redirects coming from the
- * original URL. */
+ /*
+ * Note: Fetcher will only follow meta-redirects coming from the
+ * original URL.
+ */
if (parsing && status == CrawlDatum.STATUS_FETCH_SUCCESS) {
try {
parseResult = this.parseUtil.parse(content);
} catch (Exception e) {
- LOG.warn("Error parsing: " + key + ": " + StringUtils.stringifyException(e));
+ LOG.warn("Error parsing: " + key + ": "
+ + StringUtils.stringifyException(e));
}
if (parseResult == null) {
- byte[] signature =
- SignatureFactory.getSignature(getConf()).calculate(content,
- new ParseStatus().getEmptyParse(conf));
+ byte[] signature = SignatureFactory.getSignature(getConf())
+ .calculate(content, new ParseStatus().getEmptyParse(conf));
datum.setSignature(signature);
}
}
-
- /* Store status code in content So we can read this value during
- * parsing (as a separate job) and decide to parse or not.
+
+ /*
+ * Store status code in content So we can read this value during parsing
+ * (as a separate job) and decide to parse or not.
*/
- content.getMetadata().add(Nutch.FETCH_STATUS_KEY, Integer.toString(status));
+ content.getMetadata().add(Nutch.FETCH_STATUS_KEY,
+ Integer.toString(status));
}
try {
@@ -941,7 +995,7 @@
Text url = entry.getKey();
Parse parse = entry.getValue();
ParseStatus parseStatus = parse.getData().getStatus();
-
+
if (!parseStatus.isSuccess()) {
LOG.warn("Error parsing: " + key + ": " + parseStatus);
parse = parseStatus.getEmptyParse(getConf());
@@ -949,16 +1003,16 @@
// Calculate page signature. For non-parsing fetchers this will
// be done in ParseSegment
- byte[] signature =
- SignatureFactory.getSignature(getConf()).calculate(content, parse);
+ byte[] signature = SignatureFactory.getSignature(getConf())
+ .calculate(content, parse);
// Ensure segment name and score are in parseData metadata
- parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY,
- segmentName);
- parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY,
- StringUtil.toHexString(signature));
+ parse.getData().getContentMeta()
+ .set(Nutch.SEGMENT_NAME_KEY, segmentName);
+ parse.getData().getContentMeta()
+ .set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature));
// Pass fetch time to content meta
- parse.getData().getContentMeta().set(Nutch.FETCH_TIME_KEY,
- Long.toString(datum.getFetchTime()));
+ parse.getData().getContentMeta()
+ .set(Nutch.FETCH_TIME_KEY, Long.toString(datum.getFetchTime()));
if (url.equals(key))
datum.setSignature(signature);
try {
@@ -969,15 +1023,14 @@
LOG.warn("Couldn't pass score, url " + key + " (" + e + ")");
}
}
- output.collect(url, new NutchWritable(
- new ParseImpl(new ParseText(parse.getText()),
- parse.getData(), parse.isCanonical())));
+ output.collect(url, new NutchWritable(new ParseImpl(new ParseText(
+ parse.getText()), parse.getData(), parse.isCanonical())));
}
}
} catch (IOException e) {
if (LOG.isErrorEnabled()) {
e.printStackTrace(LogUtil.getErrorStream(LOG));
- LOG.error("fetcher caught:"+e.toString());
+ LOG.error("fetcher caught:" + e.toString());
}
}
@@ -985,38 +1038,42 @@
if (parseResult != null && !parseResult.isEmpty()) {
Parse p = parseResult.get(content.getUrl());
if (p != null) {
- reporter.incrCounter("ParserStatus", ParseStatus.majorCodes[p.getData().getStatus().getMajorCode()], 1);
+ reporter.incrCounter("ParserStatus", ParseStatus.majorCodes[p
+ .getData().getStatus().getMajorCode()], 1);
return p.getData().getStatus();
}
}
return null;
}
-
+
}
- public Fetcher() { super(null); }
+ public Fetcher() {
+ super(null);
+ }
- public Fetcher(Configuration conf) { super(conf); }
+ public Fetcher(Configuration conf) {
+ super(conf);
+ }
private void updateStatus(int bytesInPage) throws IOException {
pages.incrementAndGet();
bytes.addAndGet(bytesInPage);
}
-
- private void reportStatus(int pagesLastSec, int bytesLastSec) throws IOException {
+ private void reportStatus(int pagesLastSec, int bytesLastSec)
+ throws IOException {
String status;
- long elapsed = (System.currentTimeMillis() - start)/1000;
+ long elapsed = (System.currentTimeMillis() - start) / 1000;
- float avgPagesSec = Math.round(((float)pages.get()*10)/elapsed)/10;
- float avgBytesSec = Math.round(((((float)bytes.get())*8)/1000)/elapsed);
+ float avgPagesSec = Math.round(((float) pages.get() * 10) / elapsed) / 10;
+ float avgBytesSec = Math.round(((((float) bytes.get()) * 8) / 1000)
+ / elapsed);
- status = activeThreads + " threads, " +
- fetchQueues.getQueueCount() + " queues, "+
- fetchQueues.getTotalSize() + " URLs queued, "+
- pages+" pages, "+errors+" errors, "
- + avgPagesSec + " (" + pagesLastSec + ") pages/s, "
- + avgBytesSec + " (" + bytesLastSec + ") kbits/s, ";
+ status = activeThreads + " threads, " + fetchQueues.getQueueCount()
+ + " queues, " + fetchQueues.getTotalSize() + " URLs queued, " + pages
+ + " pages, " + errors + " errors, " + avgPagesSec + " (" + pagesLastSec
+ + ") pages/s, " + avgBytesSec + " (" + bytesLastSec + ") kbits/s, ";
reporter.setStatus(status);
}
@@ -1028,12 +1085,13 @@
this.storingContent = isStoringContent(job);
this.parsing = isParsing(job);
-// if (job.getBoolean("fetcher.verbose", false)) {
-// LOG.setLevel(Level.FINE);
-// }
+ // if (job.getBoolean("fetcher.verbose", false)) {
+ // LOG.setLevel(Level.FINE);
+ // }
}
- public void close() {}
+ public void close() {
+ }
public static boolean isParsing(Configuration conf) {
return conf.getBoolean("fetcher.parse", true);
@@ -1044,41 +1102,51 @@
}
public void run(RecordReader input,
- OutputCollector output,
- Reporter reporter) throws IOException {
+ OutputCollector output, Reporter reporter)
+ throws IOException {
this.output = output;
this.reporter = reporter;
this.fetchQueues = new FetchItemQueues(getConf());
int threadCount = getConf().getInt("fetcher.threads.fetch", 10);
- if (LOG.isInfoEnabled()) { LOG.info("Fetcher: threads: " + threadCount); }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Fetcher: threads: " + threadCount);
+ }
int timeoutDivisor = getConf().getInt("fetcher.threads.timeout.divisor", 2);
- if (LOG.isInfoEnabled()) { LOG.info("Fetcher: time-out divisor: " + timeoutDivisor); }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Fetcher: time-out divisor: " + timeoutDivisor);
+ }
- int queueDepthMuliplier = getConf().getInt("fetcher.queue.depth.multiplier", 50);
-
- feeder = new QueueFeeder(input, fetchQueues, threadCount * queueDepthMuliplier);
- //feeder.setPriority((Thread.MAX_PRIORITY + Thread.NORM_PRIORITY) / 2);
-
- // the value of the time limit is either -1 or the time where it should finish
+ int queueDepthMuliplier = getConf().getInt(
+ "fetcher.queue.depth.multiplier", 50);
+
+ feeder = new QueueFeeder(input, fetchQueues, threadCount
+ * queueDepthMuliplier);
+ // feeder.setPriority((Thread.MAX_PRIORITY + Thread.NORM_PRIORITY) / 2);
+
+ // the value of the time limit is either -1 or the time where it should
+ // finish
long timelimit = getConf().getLong("fetcher.timelimit", -1);
- if (timelimit != -1) feeder.setTimeLimit(timelimit);
+ if (timelimit != -1)
+ feeder.setTimeLimit(timelimit);
feeder.start();
// set non-blocking & no-robots mode for HTTP protocol plugins.
getConf().setBoolean(Protocol.CHECK_BLOCKING, false);
getConf().setBoolean(Protocol.CHECK_ROBOTS, false);
-
- for (int i = 0; i < threadCount; i++) { // spawn threads
+
+ for (int i = 0; i < threadCount; i++) { // spawn threads
new FetcherThread(getConf()).start();
}
// select a timeout that avoids a task timeout
- long timeout = getConf().getInt("mapred.task.timeout", 10*60*1000)/timeoutDivisor;
+ long timeout = getConf().getInt("mapred.task.timeout", 10 * 60 * 1000)
+ / timeoutDivisor;
- // Used for threshold check, holds pages and bytes processed in the last second
+ // Used for threshold check, holds pages and bytes processed in the last
+ // second
int pagesLastSec;
int bytesLastSec;
@@ -1086,26 +1154,35 @@
boolean throughputThresholdExceeded = false;
int throughputThresholdNumRetries = 0;
- int throughputThresholdPages = getConf().getInt("fetcher.throughput.threshold.pages", -1);
- if (LOG.isInfoEnabled()) { LOG.info("Fetcher: throughput threshold: " + throughputThresholdPages); }
- int throughputThresholdMaxRetries = getConf().getInt("fetcher.throughput.threshold.retries", 5);
- if (LOG.isInfoEnabled()) { LOG.info("Fetcher: throughput threshold retries: " + throughputThresholdMaxRetries); }
+ int throughputThresholdPages = getConf().getInt(
+ "fetcher.throughput.threshold.pages", -1);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Fetcher: throughput threshold: " + throughputThresholdPages);
+ }
+ int throughputThresholdMaxRetries = getConf().getInt(
+ "fetcher.throughput.threshold.retries", 5);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Fetcher: throughput threshold retries: "
+ + throughputThresholdMaxRetries);
+ }
- do { // wait for threads to exit
+ do { // wait for threads to exit
pagesLastSec = pages.get();
- bytesLastSec = (int)bytes.get();
+ bytesLastSec = (int) bytes.get();
try {
Thread.sleep(1000);
- } catch (InterruptedException e) {}
+ } catch (InterruptedException e) {
+ }
pagesLastSec = pages.get() - pagesLastSec;
- bytesLastSec = (int)bytes.get() - bytesLastSec;
+ bytesLastSec = (int) bytes.get() - bytesLastSec;
reportStatus(pagesLastSec, bytesLastSec);
- LOG.info("-activeThreads=" + activeThreads + ", spinWaiting=" + spinWaiting.get()
- + ", fetchQueues.totalSize=" + fetchQueues.getTotalSize());
+ LOG.info("-activeThreads=" + activeThreads + ", spinWaiting="
+ + spinWaiting.get() + ", fetchQueues.totalSize="
+ + fetchQueues.getTotalSize());
if (!feeder.isAlive() && fetchQueues.getTotalSize() < 5) {
fetchQueues.dump();
@@ -1113,16 +1190,23 @@
// if throughput threshold is enabled
if (!feeder.isAlive() && throughputThresholdPages != -1) {
- // Have we reached the threshold of pages/second and threshold was not yet exceeded
- if (pagesLastSec > throughputThresholdPages && !throughputThresholdExceeded) {
- LOG.info("Exceding " + Integer.toString(throughputThresholdPages) + " pages/second");
+ // Have we reached the threshold of pages/second and threshold was not
+ // yet exceeded
+ if (pagesLastSec > throughputThresholdPages
+ && !throughputThresholdExceeded) {
+ LOG.info("Exceding " + Integer.toString(throughputThresholdPages)
+ + " pages/second");
throughputThresholdExceeded = true;
}
// Check if we're dropping below the threshold
- if (throughputThresholdExceeded && pagesLastSec < throughputThresholdPages) {
+ if (throughputThresholdExceeded
+ && pagesLastSec < throughputThresholdPages) {
throughputThresholdNumRetries++;
- LOG.warn(Integer.toString(throughputThresholdNumRetries) + ": dropping below configured threshold of " + Integer.toString(throughputThresholdPages) + " pages per second");
+ LOG.warn(Integer.toString(throughputThresholdNumRetries)
+ + ": dropping below configured threshold of "
+ + Integer.toString(throughputThresholdPages)
+ + " pages per second");
// Quit if we dropped below threshold too many times
if (throughputThresholdNumRetries == throughputThresholdMaxRetries) {
@@ -1131,11 +1215,13 @@
// Disable the threshold checker
throughputThresholdPages = -1;
- // Empty the queues cleanly and get number of items that were dropped
+ // Empty the queues cleanly and get number of items that were
+ // dropped
int hitByThrougputThreshold = fetchQueues.emptyQueues();
- if (hitByThrougputThreshold != 0) reporter.incrCounter("FetcherStatus",
- "hitByThrougputThreshold", hitByThrougputThreshold);
+ if (hitByThrougputThreshold != 0)
+ reporter.incrCounter("FetcherStatus", "hitByThrougputThreshold",
+ hitByThrougputThreshold);
}
}
}
@@ -1143,25 +1229,25 @@
// check timelimit
if (!feeder.isAlive()) {
int hitByTimeLimit = fetchQueues.checkTimelimit();
- if (hitByTimeLimit != 0) reporter.incrCounter("FetcherStatus",
- "hitByTimeLimit", hitByTimeLimit);
+ if (hitByTimeLimit != 0)
+ reporter.incrCounter("FetcherStatus", "hitByTimeLimit",
+ hitByTimeLimit);
}
-
+
// some requests seem to hang, despite all intentions
if ((System.currentTimeMillis() - lastRequestStart.get()) > timeout) {
if (LOG.isWarnEnabled()) {
- LOG.warn("Aborting with "+activeThreads+" hung threads.");
+ LOG.warn("Aborting with " + activeThreads + " hung threads.");
}
return;
}
} while (activeThreads.get() > 0);
LOG.info("-activeThreads=" + activeThreads);
-
+
}
- public void fetch(Path segment, int threads)
- throws IOException {
+ public void fetch(Path segment, int threads) throws IOException {
checkConfiguration();
@@ -1181,7 +1267,7 @@
LOG.info("Fetcher Timelimit set for : " + timelimit);
getConf().setLong("fetcher.timelimit", timelimit);
}
-
+
JobConf job = new NutchJob(getConf());
job.setJobName("fetch " + segment);
@@ -1191,7 +1277,8 @@
// for politeness, don't permit parallel execution of a single task
job.setSpeculativeExecution(false);
- FileInputFormat.addInputPath(job, new Path(segment, CrawlDatum.GENERATE_DIR_NAME));
+ FileInputFormat.addInputPath(job, new Path(segment,
+ CrawlDatum.GENERATE_DIR_NAME));
job.setInputFormat(InputFormat.class);
job.setMapRunnerClass(Fetcher.class);
@@ -1204,16 +1291,16 @@
JobClient.runJob(job);
long end = System.currentTimeMillis();
- LOG.info("Fetcher: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end));
+ LOG.info("Fetcher: finished at " + sdf.format(end) + ", elapsed: "
+ + TimingUtil.elapsedTime(start, end));
}
-
/** Run the fetcher. */
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(NutchConfiguration.create(), new Fetcher(), args);
System.exit(res);
}
-
+
public int run(String[] args) throws Exception {
String usage = "Usage: Fetcher [-threads n]";
@@ -1222,15 +1309,15 @@
System.err.println(usage);
return -1;
}
-
+
Path segment = new Path(args[0]);
int threads = getConf().getInt("fetcher.threads.fetch", 10);
boolean parsing = false;
- for (int i = 1; i < args.length; i++) { // parse command line
- if (args[i].equals("-threads")) { // found -threads option
- threads = Integer.parseInt(args[++i]);
+ for (int i = 1; i < args.length; i++) { // parse command line
+ if (args[i].equals("-threads")) { // found -threads option
+ threads = Integer.parseInt(args[++i]);
}
}
Index: src/java/org/apache/nutch/fetcher/FetcherOutput.java
===================================================================
--- src/java/org/apache/nutch/fetcher/FetcherOutput.java (revision 1188252)
+++ src/java/org/apache/nutch/fetcher/FetcherOutput.java (working copy)
@@ -30,10 +30,10 @@
private Content content;
private ParseImpl parse;
- public FetcherOutput() {}
+ public FetcherOutput() {
+ }
- public FetcherOutput(CrawlDatum crawlDatum, Content content,
- ParseImpl parse) {
+ public FetcherOutput(CrawlDatum crawlDatum, Content content, ParseImpl parse) {
this.crawlDatum = crawlDatum;
this.content = content;
this.parse = parse;
@@ -59,22 +59,29 @@
}
}
- public CrawlDatum getCrawlDatum() { return crawlDatum; }
- public Content getContent() { return content; }
- public ParseImpl getParse() { return parse; }
+ public CrawlDatum getCrawlDatum() {
+ return crawlDatum;
+ }
+ public Content getContent() {
+ return content;
+ }
+
+ public ParseImpl getParse() {
+ return parse;
+ }
+
public boolean equals(Object o) {
if (!(o instanceof FetcherOutput))
return false;
- FetcherOutput other = (FetcherOutput)o;
- return
- this.crawlDatum.equals(other.crawlDatum) &&
- this.content.equals(other.content);
+ FetcherOutput other = (FetcherOutput) o;
+ return this.crawlDatum.equals(other.crawlDatum)
+ && this.content.equals(other.content);
}
public String toString() {
StringBuffer buffer = new StringBuffer();
- buffer.append("CrawlDatum: " + crawlDatum+"\n" );
+ buffer.append("CrawlDatum: " + crawlDatum + "\n");
return buffer.toString();
}
Index: src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java
===================================================================
--- src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (revision 1188252)
+++ src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (working copy)
@@ -48,74 +48,68 @@
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
Path out = FileOutputFormat.getOutputPath(job);
if ((out == null) && (job.getNumReduceTasks() != 0)) {
- throw new InvalidJobConfException(
- "Output directory not set in JobConf.");
+ throw new InvalidJobConfException("Output directory not set in JobConf.");
}
if (fs == null) {
- fs = out.getFileSystem(job);
+ fs = out.getFileSystem(job);
}
if (fs.exists(new Path(out, CrawlDatum.FETCH_DIR_NAME)))
- throw new IOException("Segment already fetched!");
+ throw new IOException("Segment already fetched!");
}
public RecordWriter getRecordWriter(final FileSystem fs,
- final JobConf job,
- final String name,
- final Progressable progress) throws IOException {
+ final JobConf job, final String name, final Progressable progress)
+ throws IOException {
Path out = FileOutputFormat.getOutputPath(job);
- final Path fetch =
- new Path(new Path(out, CrawlDatum.FETCH_DIR_NAME), name);
- final Path content =
- new Path(new Path(out, Content.DIR_NAME), name);
-
- final CompressionType compType = SequenceFileOutputFormat.getOutputCompressionType(job);
+ final Path fetch = new Path(new Path(out, CrawlDatum.FETCH_DIR_NAME), name);
+ final Path content = new Path(new Path(out, Content.DIR_NAME), name);
- final MapFile.Writer fetchOut =
- new MapFile.Writer(job, fs, fetch.toString(), Text.class, CrawlDatum.class,
- compType, progress);
-
+ final CompressionType compType = SequenceFileOutputFormat
+ .getOutputCompressionType(job);
+
+ final MapFile.Writer fetchOut = new MapFile.Writer(job, fs,
+ fetch.toString(), Text.class, CrawlDatum.class, compType, progress);
+
return new RecordWriter() {
- private MapFile.Writer contentOut;
- private RecordWriter parseOut;
+ private MapFile.Writer contentOut;
+ private RecordWriter parseOut;
- {
- if (Fetcher.isStoringContent(job)) {
- contentOut = new MapFile.Writer(job, fs, content.toString(),
- Text.class, Content.class,
- compType, progress);
- }
+ {
+ if (Fetcher.isStoringContent(job)) {
+ contentOut = new MapFile.Writer(job, fs, content.toString(),
+ Text.class, Content.class, compType, progress);
+ }
- if (Fetcher.isParsing(job)) {
- parseOut = new ParseOutputFormat().getRecordWriter(fs, job, name, progress);
- }
+ if (Fetcher.isParsing(job)) {
+ parseOut = new ParseOutputFormat().getRecordWriter(fs, job, name,
+ progress);
}
+ }
- public void write(Text key, NutchWritable value)
- throws IOException {
+ public void write(Text key, NutchWritable value) throws IOException {
- Writable w = value.get();
-
- if (w instanceof CrawlDatum)
- fetchOut.append(key, w);
- else if (w instanceof Content)
- contentOut.append(key, w);
- else if (w instanceof Parse)
- parseOut.write(key, (Parse)w);
- }
+ Writable w = value.get();
- public void close(Reporter reporter) throws IOException {
- fetchOut.close();
- if (contentOut != null) {
- contentOut.close();
- }
- if (parseOut != null) {
- parseOut.close(reporter);
- }
+ if (w instanceof CrawlDatum)
+ fetchOut.append(key, w);
+ else if (w instanceof Content)
+ contentOut.append(key, w);
+ else if (w instanceof Parse)
+ parseOut.write(key, (Parse) w);
+ }
+
+ public void close(Reporter reporter) throws IOException {
+ fetchOut.close();
+ if (contentOut != null) {
+ contentOut.close();
}
+ if (parseOut != null) {
+ parseOut.close(reporter);
+ }
+ }
- };
+ };
- }
+ }
}
-
Index: src/java/org/apache/nutch/metadata/Metadata.java
===================================================================
--- src/java/org/apache/nutch/metadata/Metadata.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/Metadata.java (working copy)
@@ -27,23 +27,21 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-
/**
* A multi-valued metadata container.
- *
+ *
* @author Chris Mattmann
* @author Jérôme Charron
- *
+ *
*/
-public class Metadata implements Writable, CreativeCommons,
-DublinCore, HttpHeaders, Nutch, Office, Feed {
+public class Metadata implements Writable, CreativeCommons, DublinCore,
+ HttpHeaders, Nutch, Office, Feed {
/**
* A map of all metadata attributes.
*/
private Map metadata = null;
-
/**
* Constructs a new, empty metadata.
*/
@@ -53,9 +51,10 @@
/**
* Returns true if named value is multivalued.
- * @param name name of metadata
- * @return true is named value is multivalued, false if single
- * value or null
+ *
+ * @param name
+ * name of metadata
+ * @return true is named value is multivalued, false if single value or null
*/
public boolean isMultiValued(final String name) {
return metadata.get(name) != null && metadata.get(name).length > 1;
@@ -63,6 +62,7 @@
/**
* Returns an array of the names contained in the metadata.
+ *
* @return Metadata names
*/
public String[] names() {
@@ -70,11 +70,11 @@
}
/**
- * Get the value associated to a metadata name.
- * If many values are assiociated to the specified name, then the first
- * one is returned.
- *
- * @param name of the metadata.
+ * Get the value associated to a metadata name. If many values are assiociated
+ * to the specified name, then the first one is returned.
+ *
+ * @param name
+ * of the metadata.
* @return the value associated to the specified metadata name.
*/
public String get(final String name) {
@@ -88,13 +88,15 @@
/**
* Get the values associated to a metadata name.
- * @param name of the metadata.
+ *
+ * @param name
+ * of the metadata.
* @return the values associated to a metadata name.
*/
public String[] getValues(final String name) {
return _getValues(name);
}
-
+
private String[] _getValues(final String name) {
String[] values = metadata.get(name);
if (values == null) {
@@ -104,12 +106,13 @@
}
/**
- * Add a metadata name/value mapping.
- * Add the specified value to the list of values associated to the
- * specified metadata name.
- *
- * @param name the metadata name.
- * @param value the metadata value.
+ * Add a metadata name/value mapping. Add the specified value to the list of
+ * values associated to the specified metadata name.
+ *
+ * @param name
+ * the metadata name.
+ * @param value
+ * the metadata value.
*/
public void add(final String name, final String value) {
String[] values = metadata.get(name);
@@ -125,31 +128,37 @@
/**
* Copy All key-value pairs from properties.
- * @param properties properties to copy from
+ *
+ * @param properties
+ * properties to copy from
*/
public void setAll(Properties properties) {
Enumeration names = properties.propertyNames();
while (names.hasMoreElements()) {
String name = (String) names.nextElement();
- metadata.put(name, new String[]{properties.getProperty(name)});
+ metadata.put(name, new String[] { properties.getProperty(name) });
}
}
/**
- * Set metadata name/value.
- * Associate the specified value to the specified metadata name. If some
- * previous values were associated to this name, they are removed.
- *
- * @param name the metadata name.
- * @param value the metadata value.
+ * Set metadata name/value. Associate the specified value to the specified
+ * metadata name. If some previous values were associated to this name, they
+ * are removed.
+ *
+ * @param name
+ * the metadata name.
+ * @param value
+ * the metadata value.
*/
public void set(String name, String value) {
- metadata.put(name, new String[]{value});
+ metadata.put(name, new String[] { value });
}
/**
* Remove a metadata and all its associated values.
- * @param name metadata name to remove
+ *
+ * @param name
+ * metadata name to remove
*/
public void remove(String name) {
metadata.remove(name);
@@ -157,12 +166,13 @@
/**
* Returns the number of metadata names in this metadata.
+ *
* @return number of metadata names
*/
public int size() {
return metadata.size();
}
-
+
/** Remove all mappings from metadata. */
public void clear() {
metadata.clear();
@@ -170,7 +180,9 @@
public boolean equals(Object o) {
- if (o == null) { return false; }
+ if (o == null) {
+ return false;
+ }
Metadata other = null;
try {
@@ -179,7 +191,9 @@
return false;
}
- if (other.size() != size()) { return false; }
+ if (other.size() != size()) {
+ return false;
+ }
String[] names = names();
for (int i = 0; i < names.length; i++) {
@@ -203,10 +217,7 @@
for (int i = 0; i < names.length; i++) {
String[] values = _getValues(names[i]);
for (int j = 0; j < values.length; j++) {
- buf.append(names[i])
- .append("=")
- .append(values[j])
- .append(" ");
+ buf.append(names[i]).append("=").append(values[j]).append(" ");
}
}
return buf.toString();
Index: src/java/org/apache/nutch/metadata/Nutch.java
===================================================================
--- src/java/org/apache/nutch/metadata/Nutch.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/Nutch.java (working copy)
@@ -18,21 +18,18 @@
import org.apache.hadoop.io.Text;
-
/**
* A collection of Nutch internal metadata constants.
- *
+ *
* @author Chris Mattmann
* @author Jérôme Charron
*/
public interface Nutch {
-
- public static final String ORIGINAL_CHAR_ENCODING =
- "OriginalCharEncoding";
-
- public static final String CHAR_ENCODING_FOR_CONVERSION =
- "CharEncodingForConversion";
+ public static final String ORIGINAL_CHAR_ENCODING = "OriginalCharEncoding";
+
+ public static final String CHAR_ENCODING_FOR_CONVERSION = "CharEncodingForConversion";
+
public static final String SIGNATURE_KEY = "nutch.content.digest";
public static final String SEGMENT_NAME_KEY = "nutch.segment.name";
@@ -41,17 +38,22 @@
public static final String GENERATE_TIME_KEY = "_ngt_";
- public static final Text WRITABLE_GENERATE_TIME_KEY = new Text(GENERATE_TIME_KEY);
+ public static final Text WRITABLE_GENERATE_TIME_KEY = new Text(
+ GENERATE_TIME_KEY);
public static final String PROTO_STATUS_KEY = "_pst_";
- public static final Text WRITABLE_PROTO_STATUS_KEY = new Text(PROTO_STATUS_KEY);
-
+ public static final Text WRITABLE_PROTO_STATUS_KEY = new Text(
+ PROTO_STATUS_KEY);
+
public static final String FETCH_TIME_KEY = "_ftk_";
-
+
public static final String FETCH_STATUS_KEY = "_fst_";
- /** Sites may request that search engines don't provide access to cached documents. */
+ /**
+ * Sites may request that search engines don't provide access to cached
+ * documents.
+ */
public static final String CACHING_FORBIDDEN_KEY = "caching.forbidden";
/** Show both original forbidden content and summaries (default). */
Index: src/java/org/apache/nutch/metadata/DublinCore.java
===================================================================
--- src/java/org/apache/nutch/metadata/DublinCore.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/DublinCore.java (working copy)
@@ -16,149 +16,146 @@
*/
package org.apache.nutch.metadata;
-
/**
* A collection of Dublin Core metadata names.
- *
- * @see dublincore.org
- *
+ *
+ * @see dublincore.org
+ *
* @author Chris Mattmann
* @author Jérôme Charron
*/
public interface DublinCore {
-
-
+
/**
- * Typically, Format may include the media-type or dimensions of the
- * resource. Format may be used to determine the software, hardware or other
- * equipment needed to display or operate the resource. Examples of
- * dimensions include size and duration. Recommended best practice is to
- * select a value from a controlled vocabulary (for example, the list of
- * Internet Media Types [MIME] defining computer media formats).
+ * Typically, Format may include the media-type or dimensions of the resource.
+ * Format may be used to determine the software, hardware or other equipment
+ * needed to display or operate the resource. Examples of dimensions include
+ * size and duration. Recommended best practice is to select a value from a
+ * controlled vocabulary (for example, the list of Internet Media Types [MIME]
+ * defining computer media formats).
*/
public static final String FORMAT = "format";
-
+
/**
- * Recommended best practice is to identify the resource by means of a
- * string or number conforming to a formal identification system. Example
- * formal identification systems include the Uniform Resource Identifier
- * (URI) (including the Uniform Resource Locator (URL)), the Digital Object
+ * Recommended best practice is to identify the resource by means of a string
+ * or number conforming to a formal identification system. Example formal
+ * identification systems include the Uniform Resource Identifier (URI)
+ * (including the Uniform Resource Locator (URL)), the Digital Object
* Identifier (DOI) and the International Standard Book Number (ISBN).
*/
public static final String IDENTIFIER = "identifier";
-
+
/**
* Date on which the resource was changed.
*/
public static final String MODIFIED = "modified";
-
+
/**
* An entity responsible for making contributions to the content of the
- * resource. Examples of a Contributor include a person, an organisation, or
- * a service. Typically, the name of a Contributor should be used to
- * indicate the entity.
+ * resource. Examples of a Contributor include a person, an organisation, or a
+ * service. Typically, the name of a Contributor should be used to indicate
+ * the entity.
*/
public static final String CONTRIBUTOR = "contributor";
-
+
/**
- * The extent or scope of the content of the resource. Coverage will
- * typically include spatial location (a place name or geographic
- * coordinates), temporal period (a period label, date, or date range) or
- * jurisdiction (such as a named administrative entity). Recommended best
- * practice is to select a value from a controlled vocabulary (for example,
- * the Thesaurus of Geographic Names [TGN]) and that, where appropriate,
- * named places or time periods be used in preference to numeric identifiers
- * such as sets of coordinates or date ranges.
+ * The extent or scope of the content of the resource. Coverage will typically
+ * include spatial location (a place name or geographic coordinates), temporal
+ * period (a period label, date, or date range) or jurisdiction (such as a
+ * named administrative entity). Recommended best practice is to select a
+ * value from a controlled vocabulary (for example, the Thesaurus of
+ * Geographic Names [TGN]) and that, where appropriate, named places or time
+ * periods be used in preference to numeric identifiers such as sets of
+ * coordinates or date ranges.
*/
public static final String COVERAGE = "coverage";
-
+
/**
* An entity primarily responsible for making the content of the resource.
* Examples of a Creator include a person, an organisation, or a service.
* Typically, the name of a Creator should be used to indicate the entity.
*/
public static final String CREATOR = "creator";
-
+
/**
* A date associated with an event in the life cycle of the resource.
- * Typically, Date will be associated with the creation or availability of
- * the resource. Recommended best practice for encoding the date value is
- * defined in a profile of ISO 8601 [W3CDTF] and follows the YYYY-MM-DD
- * format.
+ * Typically, Date will be associated with the creation or availability of the
+ * resource. Recommended best practice for encoding the date value is defined
+ * in a profile of ISO 8601 [W3CDTF] and follows the YYYY-MM-DD format.
*/
public static final String DATE = "date";
-
+
/**
* An account of the content of the resource. Description may include but is
* not limited to: an abstract, table of contents, reference to a graphical
* representation of content or a free-text account of the content.
*/
public static final String DESCRIPTION = "description";
-
+
/**
* A language of the intellectual content of the resource. Recommended best
* practice is to use RFC 3066 [RFC3066], which, in conjunction with ISO 639
- * [ISO639], defines two- and three-letter primary language tags with
- * optional subtags. Examples include "en" or "eng" for English, "akk" for
- * Akkadian, and "en-GB" for English used in the United Kingdom.
+ * [ISO639], defines two- and three-letter primary language tags with optional
+ * subtags. Examples include "en" or "eng" for English, "akk" for Akkadian,
+ * and "en-GB" for English used in the United Kingdom.
*/
public static final String LANGUAGE = "language";
-
+
/**
* An entity responsible for making the resource available. Examples of a
* Publisher include a person, an organisation, or a service. Typically, the
* name of a Publisher should be used to indicate the entity.
*/
public static final String PUBLISHER = "publisher";
-
+
/**
* A reference to a related resource. Recommended best practice is to
* reference the resource by means of a string or number conforming to a
* formal identification system.
*/
public static final String RELATION = "relation";
-
+
/**
- * Information about rights held in and over the resource. Typically, a
- * Rights element will contain a rights management statement for the
- * resource, or reference a service providing such information. Rights
- * information often encompasses Intellectual Property Rights (IPR),
- * Copyright, and various Property Rights. If the Rights element is absent,
- * no assumptions can be made about the status of these and other rights
- * with respect to the resource.
+ * Information about rights held in and over the resource. Typically, a Rights
+ * element will contain a rights management statement for the resource, or
+ * reference a service providing such information. Rights information often
+ * encompasses Intellectual Property Rights (IPR), Copyright, and various
+ * Property Rights. If the Rights element is absent, no assumptions can be
+ * made about the status of these and other rights with respect to the
+ * resource.
*/
public static final String RIGHTS = "rights";
-
+
/**
* A reference to a resource from which the present resource is derived. The
* present resource may be derived from the Source resource in whole or in
- * part. Recommended best practice is to reference the resource by means of
- * a string or number conforming to a formal identification system.
+ * part. Recommended best practice is to reference the resource by means of a
+ * string or number conforming to a formal identification system.
*/
public static final String SOURCE = "source";
-
+
/**
* The topic of the content of the resource. Typically, a Subject will be
- * expressed as keywords, key phrases or classification codes that describe
- * a topic of the resource. Recommended best practice is to select a value
- * from a controlled vocabulary or formal classification scheme.
+ * expressed as keywords, key phrases or classification codes that describe a
+ * topic of the resource. Recommended best practice is to select a value from
+ * a controlled vocabulary or formal classification scheme.
*/
public static final String SUBJECT = "subject";
-
+
/**
* A name given to the resource. Typically, a Title will be a name by which
* the resource is formally known.
*/
public static final String TITLE = "title";
-
+
/**
* The nature or genre of the content of the resource. Type includes terms
- * describing general categories, functions, genres, or aggregation levels
- * for content. Recommended best practice is to select a value from a
- * controlled vocabulary (for example, the DCMI Type Vocabulary [DCMITYPE]).
- * To describe the physical or digital manifestation of the resource, use
- * the Format element.
+ * describing general categories, functions, genres, or aggregation levels for
+ * content. Recommended best practice is to select a value from a controlled
+ * vocabulary (for example, the DCMI Type Vocabulary [DCMITYPE]). To describe
+ * the physical or digital manifestation of the resource, use the Format
+ * element.
*/
public static final String TYPE = "type";
-
+
}
Index: src/java/org/apache/nutch/metadata/MetaWrapper.java
===================================================================
--- src/java/org/apache/nutch/metadata/MetaWrapper.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/MetaWrapper.java (working copy)
@@ -28,28 +28,29 @@
/**
* This is a simple decorator that adds metadata to any Writable-s that can be
* serialized by NutchWritable. This is useful when data needs to be
- * temporarily enriched during processing, but this
- * temporary metadata doesn't need to be permanently stored after the job is done.
+ * temporarily enriched during processing, but this temporary metadata doesn't
+ * need to be permanently stored after the job is done.
*
* @author Andrzej Bialecki
*/
public class MetaWrapper extends NutchWritable {
private Metadata metadata;
-
+
public MetaWrapper() {
super();
metadata = new Metadata();
}
-
+
public MetaWrapper(Writable instance, Configuration conf) {
super(instance);
metadata = new Metadata();
setConf(conf);
}
-
+
public MetaWrapper(Metadata metadata, Writable instance, Configuration conf) {
super(instance);
- if (metadata == null) metadata = new Metadata();
+ if (metadata == null)
+ metadata = new Metadata();
this.metadata = metadata;
setConf(conf);
}
@@ -60,43 +61,52 @@
public Metadata getMetadata() {
return metadata;
}
-
+
/**
- * Add metadata. See {@link Metadata#add(String, String)} for more information.
- * @param name metadata name
- * @param value metadata value
+ * Add metadata. See {@link Metadata#add(String, String)} for more
+ * information.
+ *
+ * @param name
+ * metadata name
+ * @param value
+ * metadata value
*/
public void addMeta(String name, String value) {
metadata.add(name, value);
}
-
+
/**
- * Set metadata. See {@link Metadata#set(String, String)} for more information.
+ * Set metadata. See {@link Metadata#set(String, String)} for more
+ * information.
+ *
* @param name
* @param value
*/
public void setMeta(String name, String value) {
metadata.set(name, value);
}
-
+
/**
* Get metadata. See {@link Metadata#get(String)} for more information.
+ *
* @param name
* @return metadata value
*/
public String getMeta(String name) {
return metadata.get(name);
}
-
+
/**
- * Get multiple metadata. See {@link Metadata#getValues(String)} for more information.
+ * Get multiple metadata. See {@link Metadata#getValues(String)} for more
+ * information.
+ *
* @param name
* @return multiple values
*/
public String[] getMetaValues(String name) {
return metadata.getValues(name);
}
-
+
public void readFields(DataInput in) throws IOException {
super.readFields(in);
metadata = new Metadata();
Index: src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
===================================================================
--- src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java (working copy)
@@ -33,7 +33,7 @@
/**
* Treshold divider.
- *
+ *
* threshold = searched.length() / TRESHOLD_DIVIDER;
*/
private static final int TRESHOLD_DIVIDER = 3;
@@ -52,7 +52,7 @@
// Uses following array to fill the metanames index and the
// metanames list.
- Class[] spellthese = {HttpHeaders.class};
+ Class[] spellthese = { HttpHeaders.class };
for (Class spellCheckedNames : spellthese) {
for (Field field : spellCheckedNames.getFields()) {
@@ -73,7 +73,7 @@
/**
* Normalizes String.
- *
+ *
* @param str
* the string to normalize
* @return normalized String
@@ -102,7 +102,7 @@
*
* If no matching with a well-known metadata name is found, then the original
* name is returned.
- *
+ *
* @param name
* Name to normalize
* @return normalized name
Index: src/java/org/apache/nutch/metadata/HttpHeaders.java
===================================================================
--- src/java/org/apache/nutch/metadata/HttpHeaders.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/HttpHeaders.java (working copy)
@@ -16,34 +16,33 @@
*/
package org.apache.nutch.metadata;
-
/**
* A collection of HTTP header names.
- *
- * @see Hypertext Transfer
- * Protocol -- HTTP/1.1 (RFC 2616)
- *
+ *
+ * @see Hypertext Transfer Protocol
+ * -- HTTP/1.1 (RFC 2616)
+ *
* @author Chris Mattmann
* @author Jérôme Charron
*/
public interface HttpHeaders {
public final static String CONTENT_ENCODING = "Content-Encoding";
-
+
public final static String CONTENT_LANGUAGE = "Content-Language";
public final static String CONTENT_LENGTH = "Content-Length";
-
+
public final static String CONTENT_LOCATION = "Content-Location";
-
+
public static final String CONTENT_DISPOSITION = "Content-Disposition";
public final static String CONTENT_MD5 = "Content-MD5";
-
+
public final static String CONTENT_TYPE = "Content-Type";
-
+
public final static String LAST_MODIFIED = "Last-Modified";
-
+
public final static String LOCATION = "Location";
}
Index: src/java/org/apache/nutch/metadata/Office.java
===================================================================
--- src/java/org/apache/nutch/metadata/Office.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/Office.java (working copy)
@@ -16,37 +16,36 @@
*/
package org.apache.nutch.metadata;
-
/**
* A collection of "Office" documents properties names.
- *
+ *
* @author Chris Mattmann
* @author Jérôme Charron
*/
public interface Office {
-
+
public static final String KEYWORDS = "Keywords";
-
+
public static final String COMMENTS = "Comments";
-
+
public static final String LAST_AUTHOR = "Last-Author";
-
+
public static final String APPLICATION_NAME = "Application-Name";
-
+
public static final String CHARACTER_COUNT = "Character Count";
-
+
public static final String LAST_PRINTED = "Last-Printed";
-
+
public static final String LAST_SAVED = "Last-Save-Date";
-
+
public static final String PAGE_COUNT = "Page-Count";
-
+
public static final String REVISION_NUMBER = "Revision-Number";
-
+
public static final String WORD_COUNT = "Word-Count";
-
+
public static final String TEMPLATE = "Template";
-
+
public static final String AUTHOR = "Author";
-
+
}
Index: src/java/org/apache/nutch/metadata/CreativeCommons.java
===================================================================
--- src/java/org/apache/nutch/metadata/CreativeCommons.java (revision 1188252)
+++ src/java/org/apache/nutch/metadata/CreativeCommons.java (working copy)
@@ -16,21 +16,20 @@
*/
package org.apache.nutch.metadata;
-
/**
* A collection of Creative Commons properties names.
- *
+ *
* @see creativecommons.org
- *
+ *
* @author Chris Mattmann
* @author Jérôme Charron
*/
public interface CreativeCommons {
-
+
public final static String LICENSE_URL = "License-Url";
-
+
public final static String LICENSE_LOCATION = "License-Location";
-
+
public final static String WORK_TYPE = "Work-Type";
-
+
}
Index: src/java/org/apache/nutch/tools/proxy/SegmentHandler.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/SegmentHandler.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/SegmentHandler.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -52,42 +53,54 @@
* XXX should turn this into a plugin?
*/
public class SegmentHandler extends AbstractTestbedHandler {
- private static final Logger LOG = LoggerFactory.getLogger(SegmentHandler.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SegmentHandler.class);
private Segment seg;
-
- private static HashMap protoCodes = new HashMap();
-
+
+ private static HashMap protoCodes = new HashMap();
+
static {
- protoCodes.put(ProtocolStatus.ACCESS_DENIED, HttpServletResponse.SC_UNAUTHORIZED);
- protoCodes.put(ProtocolStatus.BLOCKED, HttpServletResponse.SC_SERVICE_UNAVAILABLE);
- protoCodes.put(ProtocolStatus.EXCEPTION, HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ protoCodes.put(ProtocolStatus.ACCESS_DENIED,
+ HttpServletResponse.SC_UNAUTHORIZED);
+ protoCodes.put(ProtocolStatus.BLOCKED,
+ HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ protoCodes.put(ProtocolStatus.EXCEPTION,
+ HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
protoCodes.put(ProtocolStatus.FAILED, HttpServletResponse.SC_BAD_REQUEST);
protoCodes.put(ProtocolStatus.GONE, HttpServletResponse.SC_GONE);
- protoCodes.put(ProtocolStatus.MOVED, HttpServletResponse.SC_MOVED_PERMANENTLY);
- protoCodes.put(ProtocolStatus.NOTFETCHING, HttpServletResponse.SC_BAD_REQUEST);
+ protoCodes.put(ProtocolStatus.MOVED,
+ HttpServletResponse.SC_MOVED_PERMANENTLY);
+ protoCodes.put(ProtocolStatus.NOTFETCHING,
+ HttpServletResponse.SC_BAD_REQUEST);
protoCodes.put(ProtocolStatus.NOTFOUND, HttpServletResponse.SC_NOT_FOUND);
- protoCodes.put(ProtocolStatus.NOTMODIFIED, HttpServletResponse.SC_NOT_MODIFIED);
- protoCodes.put(ProtocolStatus.PROTO_NOT_FOUND, HttpServletResponse.SC_BAD_REQUEST);
- protoCodes.put(ProtocolStatus.REDIR_EXCEEDED, HttpServletResponse.SC_BAD_REQUEST);
+ protoCodes.put(ProtocolStatus.NOTMODIFIED,
+ HttpServletResponse.SC_NOT_MODIFIED);
+ protoCodes.put(ProtocolStatus.PROTO_NOT_FOUND,
+ HttpServletResponse.SC_BAD_REQUEST);
+ protoCodes.put(ProtocolStatus.REDIR_EXCEEDED,
+ HttpServletResponse.SC_BAD_REQUEST);
protoCodes.put(ProtocolStatus.RETRY, HttpServletResponse.SC_BAD_REQUEST);
- protoCodes.put(ProtocolStatus.ROBOTS_DENIED, HttpServletResponse.SC_FORBIDDEN);
+ protoCodes.put(ProtocolStatus.ROBOTS_DENIED,
+ HttpServletResponse.SC_FORBIDDEN);
protoCodes.put(ProtocolStatus.SUCCESS, HttpServletResponse.SC_OK);
- protoCodes.put(ProtocolStatus.TEMP_MOVED, HttpServletResponse.SC_MOVED_TEMPORARILY);
- protoCodes.put(ProtocolStatus.WOULDBLOCK, HttpServletResponse.SC_BAD_REQUEST);
+ protoCodes.put(ProtocolStatus.TEMP_MOVED,
+ HttpServletResponse.SC_MOVED_TEMPORARILY);
+ protoCodes.put(ProtocolStatus.WOULDBLOCK,
+ HttpServletResponse.SC_BAD_REQUEST);
}
-
+
private static class SegmentPathFilter implements PathFilter {
public static final SegmentPathFilter INSTANCE = new SegmentPathFilter();
-
+
@Override
public boolean accept(Path p) {
return p.getName().startsWith("part-");
}
-
+
}
-
+
private static class Segment implements Closeable {
-
+
private static final Partitioner PARTITIONER = new HashPartitioner();
private FileSystem fs;
@@ -101,7 +114,8 @@
private MapFile.Reader[] crawl;
private Configuration conf;
- public Segment(FileSystem fs, Path segmentDir, Configuration conf) throws IOException {
+ public Segment(FileSystem fs, Path segmentDir, Configuration conf)
+ throws IOException {
this.fs = fs;
this.segmentDir = segmentDir;
this.conf = conf;
@@ -112,43 +126,52 @@
if (crawl == null)
crawl = getReaders(CrawlDatum.FETCH_DIR_NAME);
}
- return (CrawlDatum)getEntry(crawl, url, new CrawlDatum());
+ return (CrawlDatum) getEntry(crawl, url, new CrawlDatum());
}
-
+
public Content getContent(Text url) throws IOException {
synchronized (cLock) {
if (content == null)
content = getReaders(Content.DIR_NAME);
}
- return (Content)getEntry(content, url, new Content());
+ return (Content) getEntry(content, url, new Content());
}
/** Open the output generated by this format. */
private MapFile.Reader[] getReaders(String subDir) throws IOException {
Path dir = new Path(segmentDir, subDir);
FileSystem fs = dir.getFileSystem(conf);
- Path[] names = FileUtil.stat2Paths(fs.listStatus(dir, SegmentPathFilter.INSTANCE));
+ Path[] names = FileUtil.stat2Paths(fs.listStatus(dir,
+ SegmentPathFilter.INSTANCE));
// sort names, so that hash partitioning works
Arrays.sort(names);
-
+
MapFile.Reader[] parts = new MapFile.Reader[names.length];
for (int i = 0; i < names.length; i++) {
parts[i] = new MapFile.Reader(fs, names[i].toString(), conf);
}
return parts;
}
-
- private Writable getEntry(MapFile.Reader[] readers, Text url,
- Writable entry) throws IOException {
+
+ private Writable getEntry(MapFile.Reader[] readers, Text url, Writable entry)
+ throws IOException {
return MapFileOutputFormat.getEntry(readers, PARTITIONER, url, entry);
}
public void close() throws IOException {
- if (content != null) { closeReaders(content); }
- if (parseText != null) { closeReaders(parseText); }
- if (parseData != null) { closeReaders(parseData); }
- if (crawl != null) { closeReaders(crawl); }
+ if (content != null) {
+ closeReaders(content);
+ }
+ if (parseText != null) {
+ closeReaders(parseText);
+ }
+ if (parseData != null) {
+ closeReaders(parseData);
+ }
+ if (crawl != null) {
+ closeReaders(crawl);
+ }
}
private void closeReaders(MapFile.Reader[] readers) throws IOException {
@@ -158,14 +181,14 @@
}
}
-
+
public SegmentHandler(Configuration conf, Path name) throws Exception {
seg = new Segment(FileSystem.get(conf), name, conf);
}
@Override
public void handle(Request req, HttpServletResponse res, String target,
- int dispatch) throws IOException, ServletException {
+ int dispatch) throws IOException, ServletException {
try {
String uri = req.getUri().toString();
LOG.info("URI: " + uri);
@@ -175,17 +198,18 @@
if (cd != null) {
addMyHeader(res, "Res", "found");
LOG.info("-got " + cd.toString());
- ProtocolStatus ps = (ProtocolStatus)cd.getMetaData().get(Nutch.WRITABLE_PROTO_STATUS_KEY);
+ ProtocolStatus ps = (ProtocolStatus) cd.getMetaData().get(
+ Nutch.WRITABLE_PROTO_STATUS_KEY);
if (ps != null) {
Integer TrCode = protoCodes.get(ps.getCode());
if (TrCode != null) {
- res.setStatus(TrCode.intValue());
+ res.setStatus(TrCode.intValue());
} else {
res.setStatus(HttpServletResponse.SC_OK);
}
addMyHeader(res, "ProtocolStatus", ps.toString());
} else {
- res.setStatus(HttpServletResponse.SC_OK);
+ res.setStatus(HttpServletResponse.SC_OK);
}
Content c = seg.getContent(url);
if (c == null) { // missing content
Index: src/java/org/apache/nutch/tools/proxy/FakeHandler.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/FakeHandler.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/FakeHandler.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -29,22 +30,20 @@
public class FakeHandler extends AbstractTestbedHandler {
Random r = new Random(1234567890L); // predictable
- private static final String testA =
- "Internet Weather Forecast Accuracy
\n" +
- "Weather forecasting is a secure and popular online presence, which is understandable. The weather affects most everyone's life, and the Internet can provide information on just about any location at any hour of the day or night. But how accurate is this information? How much can we trust it? Perhaps it is just my skeptical nature (or maybe the seeming unpredictability of nature), but I've never put much weight into weather forecasts - especially those made more than three days in advance. That skepticism progressed to a new high in the Summer of 2004, but I have only now done the research necessary to test the accuracy of online weather forecasts. First the story, then the data.
" +
- "An Internet Weather Forecast Gone Terribly Awry
" +
- "It was the Summer of 2004 and my wife and I were gearing up for a trip with another couple to Schlitterbahn in New Braunfels - one of the (if not the) best waterparks ever created. As a matter of course when embarking on a 2.5-hour drive to spend the day in a swimsuit, and given the tendency of the area for natural disasters, we checked the weather. The temperatures looked ideal and, most importantly, the chance of rain was a nice round goose egg.
";
- private static final String testB =
- "A couple of hours into our Schlitterbahn experience, we got on a bus to leave the 'old section' for the 'new section.' Along the way, clouds gathered and multiple claps of thunder sounded. 'So much for the 0% chance of rain,' I commented. By the time we got to our destination, lightning sightings had led to the slides and pools being evacuated and soon the rain began coming down in torrents - accompanied by voluminous lightning flashes. After at least a half an hour the downpour had subsided, but the lightning showed no sign of letting up, so we began heading back to our vehicles. A hundred yards into the parking lot, we passing a tree that had apparently been split in two during the storm (whether by lightning or wind, I'm not sure). Not but a few yards later, there was a distinct thud and the husband of the couple accompanying us cried out as a near racquetball sized hunk of ice rebounded off of his head and onto the concrete. Soon, similarly sized hail was falling all around us as everyone scampered for cover. Some cowered under overturned trashcans while others were more fortunate and made it indoors.
" +
- "The hail, rain and lightning eventually subsided, but the most alarming news was waiting on cell phone voicemail. A friend who lived in the area had called frantically, knowing we were at the park, as the local news was reporting multiple people had been by struck by lightning at Schlitterbahn during the storm.
" +
- "'So much for the 0% chance of rain,' I repeated.
";
+ private static final String testA = "Internet Weather Forecast Accuracy
\n"
+ + "Weather forecasting is a secure and popular online presence, which is understandable. The weather affects most everyone's life, and the Internet can provide information on just about any location at any hour of the day or night. But how accurate is this information? How much can we trust it? Perhaps it is just my skeptical nature (or maybe the seeming unpredictability of nature), but I've never put much weight into weather forecasts - especially those made more than three days in advance. That skepticism progressed to a new high in the Summer of 2004, but I have only now done the research necessary to test the accuracy of online weather forecasts. First the story, then the data.
"
+ + "An Internet Weather Forecast Gone Terribly Awry
"
+ + "It was the Summer of 2004 and my wife and I were gearing up for a trip with another couple to Schlitterbahn in New Braunfels - one of the (if not the) best waterparks ever created. As a matter of course when embarking on a 2.5-hour drive to spend the day in a swimsuit, and given the tendency of the area for natural disasters, we checked the weather. The temperatures looked ideal and, most importantly, the chance of rain was a nice round goose egg.
";
+ private static final String testB = "A couple of hours into our Schlitterbahn experience, we got on a bus to leave the 'old section' for the 'new section.' Along the way, clouds gathered and multiple claps of thunder sounded. 'So much for the 0% chance of rain,' I commented. By the time we got to our destination, lightning sightings had led to the slides and pools being evacuated and soon the rain began coming down in torrents - accompanied by voluminous lightning flashes. After at least a half an hour the downpour had subsided, but the lightning showed no sign of letting up, so we began heading back to our vehicles. A hundred yards into the parking lot, we passing a tree that had apparently been split in two during the storm (whether by lightning or wind, I'm not sure). Not but a few yards later, there was a distinct thud and the husband of the couple accompanying us cried out as a near racquetball sized hunk of ice rebounded off of his head and onto the concrete. Soon, similarly sized hail was falling all around us as everyone scampered for cover. Some cowered under overturned trashcans while others were more fortunate and made it indoors.
"
+ + "The hail, rain and lightning eventually subsided, but the most alarming news was waiting on cell phone voicemail. A friend who lived in the area had called frantically, knowing we were at the park, as the local news was reporting multiple people had been by struck by lightning at Schlitterbahn during the storm.
"
+ + "'So much for the 0% chance of rain,' I repeated.
";
@Override
- public void handle(Request req, HttpServletResponse res, String target,
- int dispatch) throws IOException, ServletException {
+ public void handle(Request req, HttpServletResponse res, String target,
+ int dispatch) throws IOException, ServletException {
HttpURI u = req.getUri();
String uri = u.toString();
- //System.err.println("-faking " + uri.toString());
+ // System.err.println("-faking " + uri.toString());
addMyHeader(res, "URI", uri);
// don't pass it down the chain
req.setHandled(true);
@@ -68,8 +67,10 @@
base = u.getPath();
}
String prefix = u.getScheme() + "://" + u.getHost();
- if (u.getPort() != 80 && u.getPort() != -1) base += ":" + u.getPort();
- if (!base.startsWith("/")) prefix += "/";
+ if (u.getPort() != 80 && u.getPort() != -1)
+ base += ":" + u.getPort();
+ if (!base.startsWith("/"))
+ prefix += "/";
prefix = prefix + base;
for (int i = 0; i < 10; i++) {
String link = "fake host " + h + "
\r\n";
+ String link = "fake host "
+ + h + "
\r\n";
os.write(link.getBytes());
}
// fake a link to the root URL
String link = "site " + u.getHost() + "
\r\n";
os.write(link.getBytes());
os.write(testB.getBytes());
res.flushBuffer();
} catch (IOException ioe) {
- }
+ }
}
}
Index: src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -31,29 +32,33 @@
import org.mortbay.jetty.Request;
public class LogDebugHandler extends AbstractTestbedHandler implements Filter {
- private static final Logger LOG = LoggerFactory.getLogger(LogDebugHandler.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(LogDebugHandler.class);
@Override
public void handle(Request req, HttpServletResponse res, String target,
- int dispatch) throws IOException, ServletException {
- LOG.info("-- " + req.getMethod() + " " + req.getUri().toString() + "\n" + req.getConnection().getRequestFields());
+ int dispatch) throws IOException, ServletException {
+ LOG.info("-- " + req.getMethod() + " " + req.getUri().toString() + "\n"
+ + req.getConnection().getRequestFields());
}
@Override
public void doFilter(ServletRequest req, ServletResponse res,
- FilterChain chain) throws IOException, ServletException {
- ((HttpServletResponse)res).addHeader("X-Handled-By", "AsyncProxyHandler");
- ((HttpServletResponse)res).addHeader("X-TestbedHandlers", "AsyncProxyHandler");
+ FilterChain chain) throws IOException, ServletException {
+ ((HttpServletResponse) res).addHeader("X-Handled-By", "AsyncProxyHandler");
+ ((HttpServletResponse) res).addHeader("X-TestbedHandlers",
+ "AsyncProxyHandler");
try {
chain.doFilter(req, res);
} catch (Throwable e) {
- ((HttpServletResponse)res).sendError(HttpServletResponse.SC_BAD_REQUEST, e.toString());
+ ((HttpServletResponse) res).sendError(HttpServletResponse.SC_BAD_REQUEST,
+ e.toString());
}
}
@Override
public void init(FilterConfig arg0) throws ServletException {
// TODO Auto-generated method stub
-
+
}
}
Index: src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -27,13 +28,13 @@
@Override
public void handle(Request req, HttpServletResponse res, String target,
- int dispatch) throws IOException, ServletException {
+ int dispatch) throws IOException, ServletException {
// don't pass it down the chain
req.setHandled(true);
res.addHeader("X-Handled-By", getClass().getSimpleName());
addMyHeader(res, "URI", req.getUri().toString());
- res.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found: " +
- req.getUri().toString());
+ res.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found: "
+ + req.getUri().toString());
}
}
Index: src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -30,16 +31,17 @@
@Override
public void handle(String target, HttpServletRequest req,
- HttpServletResponse res, int dispatch) throws IOException,
- ServletException {
- Request base_request = (req instanceof Request) ? (Request)req : HttpConnection.getCurrentConnection().getRequest();
+ HttpServletResponse res, int dispatch) throws IOException,
+ ServletException {
+ Request base_request = (req instanceof Request) ? (Request) req
+ : HttpConnection.getCurrentConnection().getRequest();
res.addHeader("X-TestbedHandlers", this.getClass().getSimpleName());
handle(base_request, res, target, dispatch);
}
-
- public abstract void handle(Request req, HttpServletResponse res, String target,
- int dispatch) throws IOException, ServletException;
-
+
+ public abstract void handle(Request req, HttpServletResponse res,
+ String target, int dispatch) throws IOException, ServletException;
+
public void addMyHeader(HttpServletResponse res, String name, String value) {
name = "X-" + this.getClass().getSimpleName() + "-" + name;
res.addHeader(name, value);
Index: src/java/org/apache/nutch/tools/proxy/DelayHandler.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/DelayHandler.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/DelayHandler.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -25,13 +26,13 @@
import org.mortbay.jetty.Request;
public class DelayHandler extends AbstractTestbedHandler {
-
+
public static final long DEFAULT_DELAY = 2000;
-
+
private int delay;
private boolean random;
private Random r;
-
+
public DelayHandler(int delay) {
if (delay < 0) {
delay = -delay;
@@ -43,13 +44,13 @@
@Override
public void handle(Request req, HttpServletResponse res, String target,
- int dispatch) throws IOException, ServletException {
+ int dispatch) throws IOException, ServletException {
try {
int del = random ? r.nextInt(delay) : delay;
Thread.sleep(del);
addMyHeader(res, "Delay", String.valueOf(del));
} catch (Exception e) {
-
+
}
}
}
Index: src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
===================================================================
--- src/java/org/apache/nutch/tools/proxy/TestbedProxy.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/proxy/TestbedProxy.java (working copy)
@@ -1,4 +1,5 @@
package org.apache.nutch.tools.proxy;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -44,17 +45,25 @@
*/
public static void main(String[] args) throws Exception {
if (args.length == 0) {
- System.err.println("TestbedProxy [-seg | -segdir ] [-port ] [-forward] [-fake] [-delay nnn] [-debug]");
- System.err.println("-seg \tpath to a single segment (can be specified multiple times)");
- System.err.println("-segdir \tpath to a parent directory of multiple segments (as above)");
- System.err.println("-port \trun the proxy on port (special permissions may be needed for ports < 1024)");
- System.err.println("-forward\tif specified, requests to all unknown urls will be passed to");
- System.err.println("\t\toriginal servers. If false (default) unknown urls generate 404 Not Found.");
- System.err.println("-delay\tdelay every response by nnn seconds. If delay is negative use a random value up to nnn");
- System.err.println("-fake\tif specified, requests to all unknown urls will succeed with fake content");
+ System.err
+ .println("TestbedProxy [-seg | -segdir ] [-port ] [-forward] [-fake] [-delay nnn] [-debug]");
+ System.err
+ .println("-seg \tpath to a single segment (can be specified multiple times)");
+ System.err
+ .println("-segdir \tpath to a parent directory of multiple segments (as above)");
+ System.err
+ .println("-port \trun the proxy on port (special permissions may be needed for ports < 1024)");
+ System.err
+ .println("-forward\tif specified, requests to all unknown urls will be passed to");
+ System.err
+ .println("\t\toriginal servers. If false (default) unknown urls generate 404 Not Found.");
+ System.err
+ .println("-delay\tdelay every response by nnn seconds. If delay is negative use a random value up to nnn");
+ System.err
+ .println("-fake\tif specified, requests to all unknown urls will succeed with fake content");
System.exit(-1);
}
-
+
Configuration conf = NutchConfiguration.create();
int port = conf.getInt("segment.proxy.port", 8181);
boolean forward = false;
@@ -62,7 +71,7 @@
boolean delay = false;
boolean debug = false;
int delayVal = 0;
-
+
HashSet segs = new HashSet();
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-segdir")) {
@@ -88,28 +97,30 @@
System.exit(-1);
}
}
-
+
// Create the server
Server server = new Server();
SocketConnector connector = new SocketConnector();
connector.setPort(port);
connector.setResolveNames(false);
server.addConnector(connector);
-
+
// create a list of handlers
HandlerList list = new HandlerList();
server.addHandler(list);
-
+
if (debug) {
LOG.info("* Added debug handler.");
list.addHandler(new LogDebugHandler());
}
-
+
if (delay) {
- LOG.info("* Added delay handler: " + (delayVal < 0 ? "random delay up to " + (-delayVal) : "constant delay of " + delayVal));
+ LOG.info("* Added delay handler: "
+ + (delayVal < 0 ? "random delay up to " + (-delayVal)
+ : "constant delay of " + delayVal));
list.addHandler(new DelayHandler(delayVal));
}
-
+
// XXX alternatively, we can add the DispatchHandler as the first one,
// XXX to activate handler plugins and redirect requests to appropriate
// XXX handlers ... Here we always load these handlers
@@ -122,7 +133,8 @@
list.addHandler(segment);
LOG.info("* Added segment handler for: " + p);
} catch (Exception e) {
- LOG.warn("Skipping segment '" + p + "': " + StringUtils.stringifyException(e));
+ LOG.warn("Skipping segment '" + p + "': "
+ + StringUtils.stringifyException(e));
}
}
if (forward) {
Index: src/java/org/apache/nutch/tools/arc/ArcRecordReader.java
===================================================================
--- src/java/org/apache/nutch/tools/arc/ArcRecordReader.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/arc/ArcRecordReader.java (working copy)
@@ -34,23 +34,29 @@
import org.apache.hadoop.util.StringUtils;
/**
- * The ArchRecordReader class provides a record reader which
- * reads records from arc files.
+ *
+ * The ArchRecordReader class provides a record reader which reads
+ * records from arc files.
+ *
*
- * Arc files are essentially tars of gzips. Each record in an arc file is
- * a compressed gzip. Multiple records are concatenated together to form a
- * complete arc. For more information on the arc file format see
- * {@link http://www.archive.org/web/researcher/ArcFileFormat.php}.
+ *
+ * Arc files are essentially tars of gzips. Each record in an arc file is a
+ * compressed gzip. Multiple records are concatenated together to form a
+ * complete arc. For more information on the arc file format see {@link http
+ * ://www.archive.org/web/researcher/ArcFileFormat.php}.
+ *
*
- * Arc files are used by the internet archive and grub projects.
+ *
+ * Arc files are used by the internet archive and grub projects.
+ *
*
* @see http://www.archive.org/
* @see http://www.grub.org/
*/
-public class ArcRecordReader
- implements RecordReader {
+public class ArcRecordReader implements RecordReader {
- public static final Logger LOG = LoggerFactory.getLogger(ArcRecordReader.class);
+ public static final Logger LOG = LoggerFactory
+ .getLogger(ArcRecordReader.class);
protected Configuration conf;
protected long splitStart = 0;
@@ -60,30 +66,32 @@
protected long fileLen = 0;
protected FSDataInputStream in;
- private static byte[] MAGIC = {(byte)0x1F, (byte)0x8B};
+ private static byte[] MAGIC = { (byte) 0x1F, (byte) 0x8B };
/**
- * Returns true if the byte array passed matches the gzip header magic
- * number.
+ *
+ * Returns true if the byte array passed matches the gzip header magic number.
+ *
*
- * @param input The byte array to check.
+ * @param input
+ * The byte array to check.
*
* @return True if the byte array matches the gzip header magic number.
*/
public static boolean isMagic(byte[] input) {
- // check for null and incorrect length
+ // check for null and incorrect length
if (input == null || input.length != MAGIC.length) {
return false;
}
-
+
// check byte by byte
for (int i = 0; i < MAGIC.length; i++) {
if (MAGIC[i] != input[i]) {
return false;
}
}
-
+
// must match
return true;
}
@@ -91,13 +99,16 @@
/**
* Constructor that sets the configuration and file split.
*
- * @param conf The job configuration.
- * @param split The file split to read from.
+ * @param conf
+ * The job configuration.
+ * @param split
+ * The file split to read from.
*
- * @throws IOException If an IO error occurs while initializing file split.
+ * @throws IOException
+ * If an IO error occurs while initializing file split.
*/
public ArcRecordReader(Configuration conf, FileSplit split)
- throws IOException {
+ throws IOException {
Path path = split.getPath();
FileSystem fs = path.getFileSystem(conf);
@@ -113,8 +124,7 @@
/**
* Closes the record reader resources.
*/
- public void close()
- throws IOException {
+ public void close() throws IOException {
this.in.close();
}
@@ -122,14 +132,15 @@
* Creates a new instance of the Text object for the key.
*/
public Text createKey() {
- return (Text)ReflectionUtils.newInstance(Text.class, conf);
+ return (Text) ReflectionUtils.newInstance(Text.class, conf);
}
/**
* Creates a new instance of the BytesWritable object for the key
*/
public BytesWritable createValue() {
- return (BytesWritable)ReflectionUtils.newInstance(BytesWritable.class, conf);
+ return (BytesWritable) ReflectionUtils.newInstance(BytesWritable.class,
+ conf);
}
/**
@@ -137,63 +148,64 @@
*
* @return The long of the current position in the file.
*/
- public long getPos()
- throws IOException {
+ public long getPos() throws IOException {
return in.getPos();
}
/**
- * Returns the percentage of progress in processing the file. This will be
+ * Returns the percentage of progress in processing the file. This will be
* represented as a float from 0 to 1 with 1 being 100% completed.
*
* @return The percentage of progress as a float from 0 to 1.
*/
- public float getProgress()
- throws IOException {
-
+ public float getProgress() throws IOException {
+
// if we haven't even started
if (splitEnd == splitStart) {
return 0.0f;
+ } else {
+ // the progress is current pos - where we started / length of the split
+ return Math.min(1.0f, (getPos() - splitStart) / (float) splitLen);
}
- else {
- // the progress is current pos - where we started / length of the split
- return Math.min(1.0f, (getPos() - splitStart) / (float)splitLen);
- }
}
/**
- * Returns true if the next record in the split is read into the key and
- * value pair. The key will be the arc record header and the values will be
- * the raw content bytes of the arc record.
+ *
+ * Returns true if the next record in the split is read into the key and value
+ * pair. The key will be the arc record header and the values will be the raw
+ * content bytes of the arc record.
+ *
*
- * @param key The record key
- * @param value The record value
+ * @param key
+ * The record key
+ * @param value
+ * The record value
*
* @return True if the next record is read.
*
- * @throws IOException If an error occurs while reading the record value.
+ * @throws IOException
+ * If an error occurs while reading the record value.
*/
- public boolean next(Text key, BytesWritable value)
- throws IOException {
+ public boolean next(Text key, BytesWritable value) throws IOException {
try {
-
+
// get the starting position on the input stream
long startRead = in.getPos();
byte[] magicBuffer = null;
-
+
// we need this loop to handle false positives in reading of gzip records
while (true) {
-
+
// while we haven't passed the end of the split
if (startRead >= splitEnd) {
return false;
}
-
+
// scanning for the gzip header
boolean foundStart = false;
while (!foundStart) {
-
+
// start at the current file position and scan for 1K at time, break
// if there is no more to read
startRead = in.getPos();
@@ -202,13 +214,13 @@
if (read < 0) {
break;
}
-
- // scan the byte array for the gzip header magic number. This happens
+
+ // scan the byte array for the gzip header magic number. This happens
// byte by byte
for (int i = 0; i < read - 1; i++) {
byte[] testMagic = new byte[2];
- System.arraycopy(magicBuffer, i, testMagic, 0, 2);
- if (isMagic(testMagic)) {
+ System.arraycopy(magicBuffer, i, testMagic, 0, 2);
+ if (isMagic(testMagic)) {
// set the next start to the current gzip header
startRead += i;
foundStart = true;
@@ -216,14 +228,14 @@
}
}
}
-
+
// seek to the start of the gzip header
in.seek(startRead);
ByteArrayOutputStream baos = null;
int totalRead = 0;
try {
-
+
// read 4K of the gzip at a time putting into a byte array
byte[] buffer = new byte[4096];
GZIPInputStream zin = new GZIPInputStream(in);
@@ -233,9 +245,8 @@
baos.write(buffer, 0, gzipRead);
totalRead += gzipRead;
}
- }
- catch (Exception e) {
-
+ } catch (Exception e) {
+
// there are times we get false positives where the gzip header exists
// but it is not an actual gzip record, so we ignore it and start
// over seeking
@@ -248,7 +259,7 @@
// change the output stream to a byte array
byte[] content = baos.toByteArray();
-
+
// the first line of the raw content in arc files is the header
int eol = 0;
for (int i = 0; i < content.length; i++) {
@@ -257,34 +268,33 @@
break;
}
}
-
+
// create the header and the raw content minus the header
String header = new String(content, 0, eol).trim();
byte[] raw = new byte[(content.length - eol) - 1];
System.arraycopy(content, eol + 1, raw, 0, raw.length);
-
+
// populate key and values with the header and raw content.
- Text keyText = (Text)key;
+ Text keyText = (Text) key;
keyText.set(header);
- BytesWritable valueBytes = (BytesWritable)value;
+ BytesWritable valueBytes = (BytesWritable) value;
valueBytes.set(raw, 0, raw.length);
- // TODO: It would be best to start at the end of the gzip read but
- // the bytes read in gzip don't match raw bytes in the file so we
- // overshoot the next header. With this current method you get
+ // TODO: It would be best to start at the end of the gzip read but
+ // the bytes read in gzip don't match raw bytes in the file so we
+ // overshoot the next header. With this current method you get
// some false positives but don't miss records.
if (startRead + 1 < fileLen) {
in.seek(startRead + 1);
}
-
+
// populated the record, now return
return true;
}
+ } catch (Exception e) {
+ LOG.equals(StringUtils.stringifyException(e));
}
- catch (Exception e) {
- LOG.equals(StringUtils.stringifyException(e));
- }
-
+
// couldn't populate the record or there is no next record to read
return false;
}
Index: src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java
===================================================================
--- src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java (working copy)
@@ -62,18 +62,22 @@
import org.apache.nutch.util.TimingUtil;
/**
- * The ArcSegmentCreator is a replacement for fetcher that will
- * take arc files as input and produce a nutch segment as output.
+ *
+ * The ArcSegmentCreator is a replacement for fetcher that will
+ * take arc files as input and produce a nutch segment as output.
+ *
*
- * Arc files are tars of compressed gzips which are produced by both the
- * internet archive project and the grub distributed crawler project.
+ *
+ * Arc files are tars of compressed gzips which are produced by both the
+ * internet archive project and the grub distributed crawler project.
+ *
*
*/
-public class ArcSegmentCreator
- extends Configured
- implements Tool, Mapper {
+public class ArcSegmentCreator extends Configured implements Tool,
+ Mapper {
- public static final Logger LOG = LoggerFactory.getLogger(ArcSegmentCreator.class);
+ public static final Logger LOG = LoggerFactory
+ .getLogger(ArcSegmentCreator.class);
public static final String URL_VERSION = "arc.url.version";
private JobConf jobConf;
private URLFilters urlFilters;
@@ -89,7 +93,9 @@
}
/**
- * Constructor that sets the job configuration.
+ *
+ * Constructor that sets the job configuration.
+ *
*
* @param conf
*/
@@ -105,17 +111,19 @@
public static synchronized String generateSegmentName() {
try {
Thread.sleep(1000);
+ } catch (Throwable t) {
}
- catch (Throwable t) {
- }
return sdf.format(new Date(System.currentTimeMillis()));
}
/**
- * Configures the job. Sets the url filters, scoring filters, url normalizers
- * and other relevant data.
+ *
+ * Configures the job. Sets the url filters, scoring filters, url normalizers
+ * and other relevant data.
+ *
*
- * @param job The job configuration.
+ * @param job
+ * The job configuration.
*/
public void configure(JobConf job) {
@@ -133,23 +141,31 @@
}
/**
- * Parses the raw content of a single record to create output. This method
- * is almost the same as the {@link org.apache.nutch.Fetcher#output} method in
- * terms of processing and output.
+ *
+ * Parses the raw content of a single record to create output. This method is
+ * almost the same as the {@link org.apache.nutch.Fetcher#output} method in
+ * terms of processing and output.
*
- * @param output The job output collector.
- * @param segmentName The name of the segment to create.
- * @param key The url of the record.
- * @param datum The CrawlDatum of the record.
- * @param content The raw content of the record
- * @param pstatus The protocol status
- * @param status The fetch status.
+ * @param output
+ * The job output collector.
+ * @param segmentName
+ * The name of the segment to create.
+ * @param key
+ * The url of the record.
+ * @param datum
+ * The CrawlDatum of the record.
+ * @param content
+ * The raw content of the record
+ * @param pstatus
+ * The protocol status
+ * @param status
+ * The fetch status.
*
* @return The result of the parse in a ParseStatus object.
*/
- private ParseStatus output(OutputCollector output, String segmentName,
- Text key, CrawlDatum datum, Content content, ProtocolStatus pstatus,
- int status) {
+ private ParseStatus output(OutputCollector output,
+ String segmentName, Text key, CrawlDatum datum, Content content,
+ ProtocolStatus pstatus, int status) {
// set the fetch status and the fetch time
datum.setStatus(status);
@@ -165,8 +181,7 @@
// add score to content metadata so that ParseSegment can pick it up.
try {
scfilters.passScoreBeforeParsing(key, datum, content);
- }
- catch (Exception e) {
+ } catch (Exception e) {
if (LOG.isWarnEnabled()) {
e.printStackTrace(LogUtil.getWarnStream(LOG));
LOG.warn("Couldn't pass score, url " + key + " (" + e + ")");
@@ -177,16 +192,15 @@
// parse the content
parseResult = this.parseUtil.parse(content);
- }
- catch (Exception e) {
+ } catch (Exception e) {
LOG.warn("Error parsing: " + key + ": "
- + StringUtils.stringifyException(e));
+ + StringUtils.stringifyException(e));
}
// set the content signature
if (parseResult == null) {
byte[] signature = SignatureFactory.getSignature(getConf()).calculate(
- content, new ParseStatus().getEmptyParse(getConf()));
+ content, new ParseStatus().getEmptyParse(getConf()));
datum.setSignature(signature);
}
@@ -195,7 +209,7 @@
output.collect(key, new NutchWritable(content));
if (parseResult != null) {
- for (Entry entry : parseResult) {
+ for (Entry entry : parseResult) {
Text url = entry.getKey();
Parse parse = entry.getValue();
ParseStatus parseStatus = parse.getData().getStatus();
@@ -205,36 +219,35 @@
parse = parseStatus.getEmptyParse(getConf());
}
- // Calculate page signature.
- byte[] signature = SignatureFactory.getSignature(getConf()).calculate(
- content, parse);
+ // Calculate page signature.
+ byte[] signature = SignatureFactory.getSignature(getConf())
+ .calculate(content, parse);
// Ensure segment name and score are in parseData metadata
- parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY,
- segmentName);
- parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY,
- StringUtil.toHexString(signature));
+ parse.getData().getContentMeta()
+ .set(Nutch.SEGMENT_NAME_KEY, segmentName);
+ parse.getData().getContentMeta()
+ .set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature));
// Pass fetch time to content meta
- parse.getData().getContentMeta().set(Nutch.FETCH_TIME_KEY,
- Long.toString(datum.getFetchTime()));
+ parse.getData().getContentMeta()
+ .set(Nutch.FETCH_TIME_KEY, Long.toString(datum.getFetchTime()));
if (url.equals(key))
datum.setSignature(signature);
try {
scfilters.passScoreAfterParsing(url, content, parse);
- }
- catch (Exception e) {
+ } catch (Exception e) {
if (LOG.isWarnEnabled()) {
e.printStackTrace(LogUtil.getWarnStream(LOG));
LOG.warn("Couldn't pass score, url " + key + " (" + e + ")");
}
}
output.collect(url, new NutchWritable(new ParseImpl(new ParseText(
- parse.getText()), parse.getData(), parse.isCanonical())));
+ parse.getText()), parse.getData(), parse.isCanonical())));
}
}
- }
- catch (IOException e) {
+ } catch (IOException e) {
if (LOG.isErrorEnabled()) {
- LOG.error("ArcSegmentCreator caught:" + StringUtils.stringifyException(e));
+ LOG.error("ArcSegmentCreator caught:"
+ + StringUtils.stringifyException(e));
}
}
@@ -246,42 +259,51 @@
}
}
}
-
+
return null;
}
/**
- * Logs any error that occurs during conversion.
+ *
+ * Logs any error that occurs during conversion.
+ *
*
- * @param url The url we are parsing.
- * @param t The error that occured.
+ * @param url
+ * The url we are parsing.
+ * @param t
+ * The error that occured.
*/
private void logError(Text url, Throwable t) {
if (LOG.isInfoEnabled()) {
- LOG.info("Conversion of " + url + " failed with: " +
- StringUtils.stringifyException(t));
+ LOG.info("Conversion of " + url + " failed with: "
+ + StringUtils.stringifyException(t));
}
}
/**
- * Runs the Map job to translate an arc record into output for Nutch
- * segments.
+ *
+ * Runs the Map job to translate an arc record into output for Nutch segments.
+ *
*
- * @param key The arc record header.
- * @param bytes The arc record raw content bytes.
- * @param output The output collecter.
- * @param reporter The progress reporter.
+ * @param key
+ * The arc record header.
+ * @param bytes
+ * The arc record raw content bytes.
+ * @param output
+ * The output collecter.
+ * @param reporter
+ * The progress reporter.
*/
public void map(Text key, BytesWritable bytes,
- OutputCollector output, Reporter reporter)
- throws IOException {
+ OutputCollector output, Reporter reporter)
+ throws IOException {
String[] headers = key.toString().split("\\s+");
String urlStr = headers[0];
String version = headers[2];
String contentType = headers[3];
-
- // arcs start with a file description. for now we ignore this as it is not
+
+ // arcs start with a file description. for now we ignore this as it is not
// a content record
if (urlStr.startsWith("filedesc://")) {
LOG.info("Ignoring file header: " + urlStr);
@@ -289,18 +311,17 @@
}
LOG.info("Processing: " + urlStr);
- // get the raw bytes from the arc file, create a new crawldatum
+ // get the raw bytes from the arc file, create a new crawldatum
Text url = new Text();
CrawlDatum datum = new CrawlDatum(CrawlDatum.STATUS_DB_FETCHED, interval,
- 1.0f);
+ 1.0f);
String segmentName = getConf().get(Nutch.SEGMENT_NAME_KEY);
// normalize and filter the urls
try {
urlStr = normalizers.normalize(urlStr, URLNormalizers.SCOPE_FETCHER);
urlStr = urlFilters.filter(urlStr); // filter the url
- }
- catch (Exception e) {
+ } catch (Exception e) {
if (LOG.isWarnEnabled()) {
LOG.warn("Skipping " + url + ":" + e);
}
@@ -315,37 +336,41 @@
// set the protocol status to success and the crawl status to success
// create the content from the normalized url and the raw bytes from
- // the arc file, TODO: currently this doesn't handle text of errors
+ // the arc file, TODO: currently this doesn't handle text of errors
// pages (i.e. 404, etc.). We assume we won't get those.
ProtocolStatus status = ProtocolStatus.STATUS_SUCCESS;
Content content = new Content(urlStr, urlStr, bytes.get(), contentType,
- new Metadata(), getConf());
-
+ new Metadata(), getConf());
+
// set the url version into the metadata
content.getMetadata().set(URL_VERSION, version);
ParseStatus pstatus = null;
pstatus = output(output, segmentName, url, datum, content, status,
- CrawlDatum.STATUS_FETCH_SUCCESS);
+ CrawlDatum.STATUS_FETCH_SUCCESS);
reporter.progress();
- }
- catch (Throwable t) { // unexpected exception
+ } catch (Throwable t) { // unexpected exception
logError(url, t);
output(output, segmentName, url, datum, null, null,
- CrawlDatum.STATUS_FETCH_RETRY);
+ CrawlDatum.STATUS_FETCH_RETRY);
}
}
}
/**
- * Creates the arc files to segments job.
+ *
+ * Creates the arc files to segments job.
+ *
*
- * @param arcFiles The path to the directory holding the arc files
- * @param segmentsOutDir The output directory for writing the segments
+ * @param arcFiles
+ * The path to the directory holding the arc files
+ * @param segmentsOutDir
+ * The output directory for writing the segments
*
- * @throws IOException If an IO error occurs while running the job.
+ * @throws IOException
+ * If an IO error occurs while running the job.
*/
public void createSegments(Path arcFiles, Path segmentsOutDir)
- throws IOException {
+ throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long start = System.currentTimeMillis();
@@ -369,17 +394,17 @@
JobClient.runJob(job);
long end = System.currentTimeMillis();
- LOG.info("ArcSegmentCreator: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end));
+ LOG.info("ArcSegmentCreator: finished at " + sdf.format(end)
+ + ", elapsed: " + TimingUtil.elapsedTime(start, end));
}
- public static void main(String args[])
- throws Exception {
- int res = ToolRunner.run(NutchConfiguration.create(), new ArcSegmentCreator(), args);
+ public static void main(String args[]) throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(),
+ new ArcSegmentCreator(), args);
System.exit(res);
}
- public int run(String[] args)
- throws Exception {
+ public int run(String[] args) throws Exception {
String usage = "Usage: ArcSegmentCreator ";
@@ -396,8 +421,7 @@
// create the segments from the arc files
createSegments(arcFiles, segmentsOutDir);
return 0;
- }
- catch (Exception e) {
+ } catch (Exception e) {
LOG.error("ArcSegmentCreator: " + StringUtils.stringifyException(e));
return -1;
}
Index: src/java/org/apache/nutch/tools/arc/ArcInputFormat.java
===================================================================
--- src/java/org/apache/nutch/tools/arc/ArcInputFormat.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/arc/ArcInputFormat.java (working copy)
@@ -30,21 +30,22 @@
/**
* A input format the reads arc files.
*/
-public class ArcInputFormat
- extends FileInputFormat {
+public class ArcInputFormat extends FileInputFormat {
/**
* Returns the RecordReader for reading the arc file.
*
- * @param split The InputSplit of the arc file to process.
- * @param job The job configuration.
- * @param reporter The progress reporter.
+ * @param split
+ * The InputSplit of the arc file to process.
+ * @param job
+ * The job configuration.
+ * @param reporter
+ * The progress reporter.
*/
public RecordReader getRecordReader(InputSplit split,
- JobConf job, Reporter reporter)
- throws IOException {
+ JobConf job, Reporter reporter) throws IOException {
reporter.setStatus(split.toString());
- return new ArcRecordReader(job, (FileSplit)split);
+ return new ArcRecordReader(job, (FileSplit) split);
}
}
Index: src/java/org/apache/nutch/tools/ResolveUrls.java
===================================================================
--- src/java/org/apache/nutch/tools/ResolveUrls.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/ResolveUrls.java (working copy)
@@ -59,8 +59,7 @@
/**
* A Thread which gets the ip address of a single host by name.
*/
- private static class ResolverThread
- extends Thread {
+ private static class ResolverThread extends Thread {
private String url = null;
@@ -74,14 +73,13 @@
String host = URLUtil.getHost(url);
long start = System.currentTimeMillis();
try {
-
- // get the address by name and if no error is thrown then it
+
+ // get the address by name and if no error is thrown then it
// is resolved successfully
InetAddress ia = InetAddress.getByName(host);
LOG.info("Resolved: " + host);
numResolved.incrementAndGet();
- }
- catch (Exception uhe) {
+ } catch (Exception uhe) {
LOG.info("Error Resolving: " + host);
numErrored.incrementAndGet();
}
@@ -93,8 +91,8 @@
}
/**
- * Creates a thread pool for resolving urls. Reads in the url file on the
- * local filesystem. For each url it attempts to resolve it keeping a total
+ * Creates a thread pool for resolving urls. Reads in the url file on the
+ * local filesystem. For each url it attempts to resolve it keeping a total
* account of the number resolved, errored, and the amount of time.
*/
public void resolveUrls() {
@@ -103,13 +101,14 @@
// create a thread pool with a fixed number of threads
pool = Executors.newFixedThreadPool(numThreads);
-
- // read in the urls file and loop through each line, one url per line
+
+ // read in the urls file and loop through each line, one url per
+ // line
BufferedReader buffRead = new BufferedReader(new FileReader(new File(
- urlsFile)));
+ urlsFile)));
String urlStr = null;
while ((urlStr = buffRead.readLine()) != null) {
-
+
// spin up a resolver thread per url
LOG.info("Starting: " + urlStr);
pool.execute(new ResolverThread(urlStr));
@@ -119,9 +118,8 @@
// the thread pool to give urls time to finish resolving
buffRead.close();
pool.awaitTermination(60, TimeUnit.SECONDS);
- }
- catch (Exception e) {
-
+ } catch (Exception e) {
+
// on error shutdown the thread pool immediately
pool.shutdownNow();
LOG.info(StringUtils.stringifyException(e));
@@ -129,15 +127,16 @@
// shutdown the thread pool and log totals
pool.shutdown();
- LOG.info("Total: " + numTotal.get() + ", Resovled: "
- + numResolved.get() + ", Errored: " + numErrored.get()
- + ", Average Time: " + totalTime.get() / numTotal.get());
+ LOG.info("Total: " + numTotal.get() + ", Resovled: " + numResolved.get()
+ + ", Errored: " + numErrored.get() + ", Average Time: "
+ + totalTime.get() / numTotal.get());
}
/**
* Create a new ResolveUrls with a file from the local file system.
- *
- * @param urlsFile The local urls file, one url per line.
+ *
+ * @param urlsFile
+ * The local urls file, one url per line.
*/
public ResolveUrls(String urlsFile) {
this(urlsFile, 100);
@@ -145,10 +144,12 @@
/**
* Create a new ResolveUrls with a urls file and a number of threads for the
- * Thread pool. Number of threads is 100 by default.
+ * Thread pool. Number of threads is 100 by default.
*
- * @param urlsFile The local urls file, one url per line.
- * @param numThreads The number of threads used to resolve urls in parallel.
+ * @param urlsFile
+ * The local urls file, one url per line.
+ * @param numThreads
+ * The number of threads used to resolve urls in parallel.
*/
public ResolveUrls(String urlsFile, int numThreads) {
this.urlsFile = urlsFile;
@@ -161,12 +162,12 @@
public static void main(String[] args) {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option urlOpts = OptionBuilder.withArgName("urls").hasArg().withDescription(
- "the urls file to check").create("urls");
- Option numThreadOpts = OptionBuilder.withArgName("numThreads").hasArgs().withDescription(
- "the number of threads to use").create("numThreads");
+ Option helpOpts = OptionBuilder.withArgName("help")
+ .withDescription("show this help message").create("help");
+ Option urlOpts = OptionBuilder.withArgName("urls").hasArg()
+ .withDescription("the urls file to check").create("urls");
+ Option numThreadOpts = OptionBuilder.withArgName("numThreads").hasArgs()
+ .withDescription("the number of threads to use").create("numThreads");
options.addOption(helpOpts);
options.addOption(urlOpts);
options.addOption(numThreadOpts);
@@ -191,8 +192,7 @@
}
ResolveUrls resolve = new ResolveUrls(urls, numThreads);
resolve.resolveUrls();
- }
- catch (Exception e) {
+ } catch (Exception e) {
LOG.error("ResolveUrls: " + StringUtils.stringifyException(e));
}
}
Index: src/java/org/apache/nutch/tools/DmozParser.java
===================================================================
--- src/java/org/apache/nutch/tools/DmozParser.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/DmozParser.java (working copy)
@@ -36,16 +36,15 @@
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
-
/** Utility that converts DMOZ RDF into a flat file of URLs to be injected. */
public class DmozParser {
public static final Logger LOG = LoggerFactory.getLogger(DmozParser.class);
-
- long pages = 0;
+ long pages = 0;
+
/**
- * This filter fixes characters that might offend our parser.
- * This lets us be tolerant of errors that might appear in the input XML.
+ * This filter fixes characters that might offend our parser. This lets us be
+ * tolerant of errors that might appear in the input XML.
*/
private static class XMLCharFilter extends FilterReader {
private boolean lastBad = false;
@@ -57,9 +56,9 @@
public int read() throws IOException {
int c = in.read();
int value = c;
- if (c != -1 && !(XMLChar.isValid(c))) // fix invalid characters
+ if (c != -1 && !(XMLChar.isValid(c))) // fix invalid characters
value = 'X';
- else if (lastBad && c == '<') { // fix mis-matched brackets
+ else if (lastBad && c == '<') { // fix mis-matched brackets
in.mark(1);
if (in.read() != '/')
value = 'X';
@@ -70,37 +69,35 @@
return value;
}
- public int read(char[] cbuf, int off, int len)
- throws IOException {
+ public int read(char[] cbuf, int off, int len) throws IOException {
int n = in.read(cbuf, off, len);
if (n != -1) {
for (int i = 0; i < n; i++) {
- char c = cbuf[off+i];
+ char c = cbuf[off + i];
char value = c;
- if (!(XMLChar.isValid(c))) // fix invalid characters
+ if (!(XMLChar.isValid(c))) // fix invalid characters
value = 'X';
- else if (lastBad && c == '<') { // fix mis-matched brackets
- if (i != n-1 && cbuf[off+i+1] != '/')
+ else if (lastBad && c == '<') { // fix mis-matched brackets
+ if (i != n - 1 && cbuf[off + i + 1] != '/')
value = 'X';
}
lastBad = (c == 65533);
- cbuf[off+i] = value;
+ cbuf[off + i] = value;
}
}
return n;
}
}
-
/**
- * The RDFProcessor receives tag messages during a parse
- * of RDF XML data. We build whatever structures we need
- * from these messages.
+ * The RDFProcessor receives tag messages during a parse of RDF XML data. We
+ * build whatever structures we need from these messages.
*/
private class RDFProcessor extends DefaultHandler {
String curURL = null, curSection = null;
- boolean titlePending = false, descPending = false, insideAdultSection = false;
- Pattern topicPattern = null;
+ boolean titlePending = false, descPending = false,
+ insideAdultSection = false;
+ Pattern topicPattern = null;
StringBuffer title = new StringBuffer(), desc = new StringBuffer();
XMLReader reader;
int subsetDenom;
@@ -109,10 +106,12 @@
Locator location;
/**
- * Pass in an XMLReader, plus a flag as to whether we
- * should include adult material.
+ * Pass in an XMLReader, plus a flag as to whether we should include adult
+ * material.
*/
- public RDFProcessor(XMLReader reader, int subsetDenom, boolean includeAdult, int skew, Pattern topicPattern) throws IOException {
+ public RDFProcessor(XMLReader reader, int subsetDenom,
+ boolean includeAdult, int skew, Pattern topicPattern)
+ throws IOException {
this.reader = reader;
this.subsetDenom = subsetDenom;
this.includeAdult = includeAdult;
@@ -128,20 +127,21 @@
/**
* Start of an XML elt
*/
- public void startElement(String namespaceURI, String localName, String qName, Attributes atts) throws SAXException {
+ public void startElement(String namespaceURI, String localName,
+ String qName, Attributes atts) throws SAXException {
if ("Topic".equals(qName)) {
curSection = atts.getValue("r:id");
} else if ("ExternalPage".equals(qName)) {
// Porn filter
- if ((! includeAdult) && curSection.startsWith("Top/Adult")) {
+ if ((!includeAdult) && curSection.startsWith("Top/Adult")) {
return;
}
-
+
if (topicPattern != null && !topicPattern.matcher(curSection).matches()) {
return;
}
- // Subset denominator filter.
+ // Subset denominator filter.
// Only emit with a chance of 1/denominator.
String url = atts.getValue("about");
int hashValue = MD5Hash.digest(url).hashCode();
@@ -174,18 +174,18 @@
* Termination of XML elt
*/
public void endElement(String namespaceURI, String localName, String qName)
- throws SAXException {
+ throws SAXException {
if (curURL != null) {
if ("ExternalPage".equals(qName)) {
//
- // Inc the number of pages, insert the page, and
+ // Inc the number of pages, insert the page, and
// possibly print status.
//
- System.out.println(curURL);
+ System.out.println(curURL);
pages++;
//
- // Clear out the link text. This is what
+ // Clear out the link text. This is what
// you would use for adding to the linkdb.
//
if (title.length() > 0) {
@@ -220,15 +220,13 @@
}
/**
- * From time to time the Parser will set the "current location"
- * by calling this function. It's useful for emitting locations
- * for error messages.
+ * From time to time the Parser will set the "current location" by calling
+ * this function. It's useful for emitting locations for error messages.
*/
public void setDocumentLocator(Locator locator) {
location = locator;
}
-
//
// Interface ErrorHandler
//
@@ -249,12 +247,12 @@
public void errorError(SAXParseException spe) {
if (LOG.isErrorEnabled()) {
LOG.error("Fatal err: " + spe.toString() + ": " + spe.getMessage());
- LOG.error("Last known line is " + location.getLineNumber() +
- ", column " + location.getColumnNumber());
+ LOG.error("Last known line is " + location.getLineNumber()
+ + ", column " + location.getColumnNumber());
spe.printStackTrace(LogUtil.getErrorStream(LOG));
}
}
-
+
/**
* Emit exception warning message
*/
@@ -267,34 +265,33 @@
}
/**
- * Iterate through all the items in this structured DMOZ file.
- * Add each URL to the web db.
+ * Iterate through all the items in this structured DMOZ file. Add each URL to
+ * the web db.
*/
public void parseDmozFile(File dmozFile, int subsetDenom,
- boolean includeAdult,
- int skew,
- Pattern topicPattern)
+ boolean includeAdult, int skew, Pattern topicPattern)
- throws IOException, SAXException, ParserConfigurationException {
+ throws IOException, SAXException, ParserConfigurationException {
SAXParserFactory parserFactory = SAXParserFactory.newInstance();
SAXParser parser = parserFactory.newSAXParser();
XMLReader reader = parser.getXMLReader();
// Create our own processor to receive SAX events
- RDFProcessor rp =
- new RDFProcessor(reader, subsetDenom, includeAdult,
- skew, topicPattern);
+ RDFProcessor rp = new RDFProcessor(reader, subsetDenom, includeAdult, skew,
+ topicPattern);
reader.setContentHandler(rp);
reader.setErrorHandler(rp);
LOG.info("skew = " + rp.hashSkew);
//
- // Open filtered text stream. The TextFilter makes sure that
+ // Open filtered text stream. The TextFilter makes sure that
// only appropriate XML-approved Text characters are received.
// Any non-conforming characters are silently skipped.
//
- XMLCharFilter in = new XMLCharFilter(new BufferedReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(dmozFile)), "UTF-8")));
+ XMLCharFilter in = new XMLCharFilter(new BufferedReader(
+ new InputStreamReader(new BufferedInputStream(new FileInputStream(
+ dmozFile)), "UTF-8")));
try {
InputSource is = new InputSource(in);
reader.parse(is);
@@ -309,18 +306,17 @@
}
}
- private static void addTopicsFromFile(String topicFile,
- Vector topics)
- throws IOException {
+ private static void addTopicsFromFile(String topicFile, Vector topics)
+ throws IOException {
BufferedReader in = null;
try {
- in = new BufferedReader(new InputStreamReader(new FileInputStream(topicFile), "UTF-8"));
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(
+ topicFile), "UTF-8"));
String line = null;
while ((line = in.readLine()) != null) {
topics.addElement(new String(line));
}
- }
- catch (Exception e) {
+ } catch (Exception e) {
if (LOG.isErrorEnabled()) {
LOG.error(e.toString());
e.printStackTrace(LogUtil.getErrorStream(LOG));
@@ -330,18 +326,19 @@
in.close();
}
}
-
+
/**
- * Command-line access. User may add URLs via a flat text file
- * or the structured DMOZ file. By default, we ignore Adult
- * material (as categorized by DMOZ).
+ * Command-line access. User may add URLs via a flat text file or the
+ * structured DMOZ file. By default, we ignore Adult material (as categorized
+ * by DMOZ).
*/
public static void main(String argv[]) throws Exception {
if (argv.length < 1) {
- System.err.println("Usage: DmozParser [-subset ] [-includeAdultMaterial] [-skew skew] [-topicFile ] [-topic [-topic [...]]]");
+ System.err
+ .println("Usage: DmozParser [-subset ] [-includeAdultMaterial] [-skew skew] [-topicFile ] [-topic [-topic [...]]]");
return;
}
-
+
//
// Parse the command line, figure out what kind of
// URL file we need to load
@@ -350,9 +347,9 @@
int skew = 0;
String dmozFile = argv[0];
boolean includeAdult = false;
- Pattern topicPattern = null;
+ Pattern topicPattern = null;
Vector topics = new Vector();
-
+
Configuration conf = NutchConfiguration.create();
FileSystem fs = FileSystem.get(conf);
try {
@@ -360,16 +357,16 @@
if ("-includeAdultMaterial".equals(argv[i])) {
includeAdult = true;
} else if ("-subset".equals(argv[i])) {
- subsetDenom = Integer.parseInt(argv[i+1]);
+ subsetDenom = Integer.parseInt(argv[i + 1]);
i++;
} else if ("-topic".equals(argv[i])) {
- topics.addElement(argv[i+1]);
+ topics.addElement(argv[i + 1]);
i++;
} else if ("-topicFile".equals(argv[i])) {
- addTopicsFromFile(argv[i+1], topics);
+ addTopicsFromFile(argv[i + 1], topics);
i++;
} else if ("-skew".equals(argv[i])) {
- skew = Integer.parseInt(argv[i+1]);
+ skew = Integer.parseInt(argv[i + 1]);
i++;
}
}
@@ -377,21 +374,21 @@
DmozParser parser = new DmozParser();
if (!topics.isEmpty()) {
- String regExp = new String("^(");
+ String regExp = new String("^(");
int j = 0;
- for ( ; j < topics.size() - 1; ++j) {
+ for (; j < topics.size() - 1; ++j) {
regExp = regExp.concat(topics.get(j));
regExp = regExp.concat("|");
}
regExp = regExp.concat(topics.get(j));
- regExp = regExp.concat(").*");
+ regExp = regExp.concat(").*");
LOG.info("Topic selection pattern = " + regExp);
- topicPattern = Pattern.compile(regExp);
+ topicPattern = Pattern.compile(regExp);
}
- parser.parseDmozFile(new File(dmozFile), subsetDenom,
- includeAdult, skew, topicPattern);
-
+ parser.parseDmozFile(new File(dmozFile), subsetDenom, includeAdult, skew,
+ topicPattern);
+
} finally {
fs.close();
}
Index: src/java/org/apache/nutch/tools/CrawlDBScanner.java
===================================================================
--- src/java/org/apache/nutch/tools/CrawlDBScanner.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/CrawlDBScanner.java (working copy)
@@ -57,17 +57,21 @@
*/
public class CrawlDBScanner extends Configured implements Tool,
- Mapper, Reducer {
+ Mapper,
+ Reducer {
- public static final Logger LOG = LoggerFactory.getLogger(CrawlDBScanner.class);
+ public static final Logger LOG = LoggerFactory
+ .getLogger(CrawlDBScanner.class);
- public CrawlDBScanner() {}
+ public CrawlDBScanner() {
+ }
public CrawlDBScanner(Configuration conf) {
setConf(conf);
}
- public void close() {}
+ public void close() {
+ }
private String regex = null;
private String status = null;
@@ -78,11 +82,14 @@
}
public void map(Text url, CrawlDatum crawlDatum,
- OutputCollector output, Reporter reporter) throws IOException {
+ OutputCollector output, Reporter reporter)
+ throws IOException {
// check status
if (status != null
- && !status.equalsIgnoreCase(CrawlDatum.getStatusName(crawlDatum.getStatus()))) return;
+ && !status.equalsIgnoreCase(CrawlDatum.getStatusName(crawlDatum
+ .getStatus())))
+ return;
// if URL matched regexp dump it
if (url.toString().matches(regex)) {
@@ -91,7 +98,8 @@
}
public void reduce(Text key, Iterator values,
- OutputCollector output, Reporter reporter) throws IOException {
+ OutputCollector output, Reporter reporter)
+ throws IOException {
while (values.hasNext()) {
CrawlDatum val = values.next();
output.collect(key, val);
@@ -110,7 +118,8 @@
job.setJobName("Scan : " + crawlDb + " for URLS matching : " + regex);
job.set("CrawlDBScanner.regex", regex);
- if (status != null) job.set("CrawlDBScanner.status", status);
+ if (status != null)
+ job.set("CrawlDBScanner.status", status);
FileInputFormat.addInputPath(job, new Path(crawlDb, CrawlDb.CURRENT_NAME));
job.setInputFormat(SequenceFileInputFormat.class);
@@ -147,11 +156,13 @@
}
long end = System.currentTimeMillis();
- LOG.info("CrawlDb scanner: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end));
+ LOG.info("CrawlDb scanner: finished at " + sdf.format(end) + ", elapsed: "
+ + TimingUtil.elapsedTime(start, end));
}
public static void main(String args[]) throws Exception {
- int res = ToolRunner.run(NutchConfiguration.create(), new CrawlDBScanner(), args);
+ int res = ToolRunner.run(NutchConfiguration.create(), new CrawlDBScanner(),
+ args);
System.exit(res);
}
Index: src/java/org/apache/nutch/tools/FreeGenerator.java
===================================================================
--- src/java/org/apache/nutch/tools/FreeGenerator.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/FreeGenerator.java (working copy)
@@ -54,21 +54,22 @@
import org.apache.nutch.util.TimingUtil;
/**
- * This tool generates fetchlists (segments to be fetched) from plain text
- * files containing one URL per line. It's useful when arbitrary URL-s need to
- * be fetched without adding them first to the CrawlDb, or during testing.
+ * This tool generates fetchlists (segments to be fetched) from plain text files
+ * containing one URL per line. It's useful when arbitrary URL-s need to be
+ * fetched without adding them first to the CrawlDb, or during testing.
*
* @author Andrzej Bialecki
*/
public class FreeGenerator extends Configured implements Tool {
- private static final Logger LOG = LoggerFactory.getLogger(FreeGenerator.class);
-
+ private static final Logger LOG = LoggerFactory
+ .getLogger(FreeGenerator.class);
+
private static final String FILTER_KEY = "free.generator.filter";
private static final String NORMALIZE_KEY = "free.generator.normalize";
- public static class FG extends MapReduceBase
- implements Mapper,
- Reducer {
+ public static class FG extends MapReduceBase implements
+ Mapper,
+ Reducer {
private URLNormalizers normalizers = null;
private URLFilters filters = null;
private ScoringFilters scfilters;
@@ -86,16 +87,18 @@
normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
}
}
-
+
Generator.SelectorEntry entry = new Generator.SelectorEntry();
- public void map(WritableComparable key, Text value, OutputCollector output, Reporter reporter) throws IOException {
+ public void map(WritableComparable key, Text value,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
// value is a line of text
String urlString = value.toString();
try {
if (normalizers != null) {
- urlString = normalizers.normalize(urlString, URLNormalizers.SCOPE_INJECT);
+ urlString = normalizers.normalize(urlString,
+ URLNormalizers.SCOPE_INJECT);
}
if (urlString != null && filters != null) {
urlString = filters.filter(urlString);
@@ -105,7 +108,8 @@
scfilters.injectedScore(url, datum);
}
} catch (Exception e) {
- LOG.warn("Error adding url '" + value.toString() + "', skipping: " + StringUtils.stringifyException(e));
+ LOG.warn("Error adding url '" + value.toString() + "', skipping: "
+ + StringUtils.stringifyException(e));
return;
}
if (urlString == null) {
@@ -120,11 +124,13 @@
}
public void reduce(Text key, Iterator values,
- OutputCollector output, Reporter reporter) throws IOException {
- // pick unique urls from values - discard the reduce key due to hash collisions
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ // pick unique urls from values - discard the reduce key due to hash
+ // collisions
HashMap unique = new HashMap();
while (values.hasNext()) {
- Generator.SelectorEntry entry = (Generator.SelectorEntry)values.next();
+ Generator.SelectorEntry entry = (Generator.SelectorEntry) values.next();
unique.put(entry.url, entry.datum);
}
// output unique urls
@@ -133,15 +139,20 @@
}
}
}
-
+
public int run(String[] args) throws Exception {
if (args.length < 2) {
- System.err.println("Usage: FreeGenerator [-filter] [-normalize]");
- System.err.println("\tinputDir\tinput directory containing one or more input files.");
- System.err.println("\t\tEach text file contains a list of URLs, one URL per line");
- System.err.println("\tsegmentsDir\toutput directory, where new segment will be created");
+ System.err
+ .println("Usage: FreeGenerator [-filter] [-normalize]");
+ System.err
+ .println("\tinputDir\tinput directory containing one or more input files.");
+ System.err
+ .println("\t\tEach text file contains a list of URLs, one URL per line");
+ System.err
+ .println("\tsegmentsDir\toutput directory, where new segment will be created");
System.err.println("\t-filter\trun current URLFilters on input URLs");
- System.err.println("\t-normalize\trun current URLNormalizers on input URLs");
+ System.err
+ .println("\t-normalize\trun current URLNormalizers on input URLs");
return -1;
}
boolean filter = false;
@@ -158,7 +169,7 @@
}
}
}
-
+
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long start = System.currentTimeMillis();
LOG.info("FreeGenerator: starting at " + sdf.format(start));
@@ -179,8 +190,8 @@
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(CrawlDatum.class);
job.setOutputKeyComparatorClass(Generator.HashComparator.class);
- FileOutputFormat.setOutputPath(job, new Path(args[1],
- new Path(segName, CrawlDatum.GENERATE_DIR_NAME)));
+ FileOutputFormat.setOutputPath(job, new Path(args[1], new Path(segName,
+ CrawlDatum.GENERATE_DIR_NAME)));
try {
JobClient.runJob(job);
} catch (Exception e) {
@@ -188,12 +199,14 @@
return -1;
}
long end = System.currentTimeMillis();
- LOG.info("FreeGenerator: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end));
+ LOG.info("FreeGenerator: finished at " + sdf.format(end) + ", elapsed: "
+ + TimingUtil.elapsedTime(start, end));
return 0;
}
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(NutchConfiguration.create(), new FreeGenerator(), args);
+ int res = ToolRunner.run(NutchConfiguration.create(), new FreeGenerator(),
+ args);
System.exit(res);
}
}
Index: src/java/org/apache/nutch/tools/Benchmark.java
===================================================================
--- src/java/org/apache/nutch/tools/Benchmark.java (revision 1188252)
+++ src/java/org/apache/nutch/tools/Benchmark.java (working copy)
@@ -52,13 +52,14 @@
int res = ToolRunner.run(conf, new Benchmark(), args);
System.exit(res);
}
-
+
private static String getDate() {
- return new SimpleDateFormat("yyyyMMddHHmmss").format
- (new Date(System.currentTimeMillis()));
+ return new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System
+ .currentTimeMillis()));
}
-
- private void createSeeds(FileSystem fs, Path seedsDir, int count) throws Exception {
+
+ private void createSeeds(FileSystem fs, Path seedsDir, int count)
+ throws Exception {
OutputStream os = fs.create(new Path(seedsDir, "seeds"));
for (int i = 0; i < count; i++) {
String url = "http://www.test-" + i + ".com/\r\n";
@@ -67,9 +68,9 @@
os.flush();
os.close();
}
-
+
public static final class BenchmarkResults {
- Map> timings = new HashMap>();
+ Map> timings = new HashMap>();
List runs = new ArrayList();
List stages = new ArrayList();
int seeds, depth, threads;
@@ -77,7 +78,7 @@
long topN;
long elapsed;
String plugins;
-
+
public void addTiming(String stage, String run, long timing) {
if (!runs.contains(run)) {
runs.add(run);
@@ -85,14 +86,14 @@
if (!stages.contains(stage)) {
stages.add(stage);
}
- Map t = timings.get(stage);
+ Map t = timings.get(stage);
if (t == null) {
- t = new HashMap();
+ t = new HashMap();
timings.put(stage, t);
}
t.put(run, timing);
}
-
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("* Plugins:\t" + plugins + "\n");
@@ -103,8 +104,9 @@
sb.append("* Delete:\t" + delete + "\n");
sb.append("* TOTAL ELAPSED:\t" + elapsed + "\n");
for (String stage : stages) {
- Map timing = timings.get(stage);
- if (timing == null) continue;
+ Map timing = timings.get(stage);
+ if (timing == null)
+ continue;
sb.append("- stage: " + stage + "\n");
for (String r : runs) {
Long Time = timing.get(r);
@@ -116,15 +118,16 @@
}
return sb.toString();
}
-
+
public List getStages() {
return stages;
}
+
public List getRuns() {
return runs;
}
}
-
+
public int run(String[] args) throws Exception {
String plugins = "protocol-http|parse-tika|scoring-opic|urlfilter-regex|urlnormalizer-pass";
int seeds = 1;
@@ -132,17 +135,24 @@
int threads = 10;
boolean delete = true;
long topN = Long.MAX_VALUE;
-
+
if (args.length == 0) {
- System.err.println("Usage: Benchmark [-seeds NN] [-depth NN] [-threads NN] [-keep] [-maxPerHost NN] [-plugins ]");
- System.err.println("\t-seeds NN\tcreate NN unique hosts in a seed list (default: 1)");
+ System.err
+ .println("Usage: Benchmark [-seeds NN] [-depth NN] [-threads NN] [-keep] [-maxPerHost NN] [-plugins ]");
+ System.err
+ .println("\t-seeds NN\tcreate NN unique hosts in a seed list (default: 1)");
System.err.println("\t-depth NN\tperform NN crawl cycles (default: 10)");
- System.err.println("\t-threads NN\tuse NN threads per Fetcher task (default: 10)");
- System.err.println("\t-keep\tkeep segment data (default: delete after updatedb)");
+ System.err
+ .println("\t-threads NN\tuse NN threads per Fetcher task (default: 10)");
+ System.err
+ .println("\t-keep\tkeep segment data (default: delete after updatedb)");
System.err.println("\t-plugins \toverride 'plugin.includes'.");
- System.err.println("\tNOTE: if not specified, this is reset to: " + plugins);
- System.err.println("\tNOTE: if 'default' is specified then a value set in nutch-default/nutch-site is used.");
- System.err.println("\t-maxPerHost NN\tmax. # of URLs per host in a fetchlist");
+ System.err.println("\tNOTE: if not specified, this is reset to: "
+ + plugins);
+ System.err
+ .println("\tNOTE: if 'default' is specified then a value set in nutch-default/nutch-site is used.");
+ System.err
+ .println("\t-maxPerHost NN\tmax. # of URLs per host in a fetchlist");
return -1;
}
int maxPerHost = Integer.MAX_VALUE;
@@ -164,13 +174,15 @@
return -1;
}
}
- BenchmarkResults res = benchmark(seeds, depth, threads, maxPerHost, topN, delete, plugins);
+ BenchmarkResults res = benchmark(seeds, depth, threads, maxPerHost, topN,
+ delete, plugins);
System.out.println(res);
return 0;
}
-
- public BenchmarkResults benchmark(int seeds, int depth, int threads, int maxPerHost,
- long topN, boolean delete, String plugins) throws Exception {
+
+ public BenchmarkResults benchmark(int seeds, int depth, int threads,
+ int maxPerHost, long topN, boolean delete, String plugins)
+ throws Exception {
Configuration conf = getConf();
conf.set("http.proxy.host", "localhost");
conf.setInt("http.proxy.port", 8181);
@@ -180,11 +192,12 @@
conf.set("plugin.includes", plugins);
}
conf.setInt(Generator.GENERATOR_MAX_COUNT, maxPerHost);
- conf.set(Generator.GENERATOR_COUNT_MODE, Generator.GENERATOR_COUNT_VALUE_HOST);
- JobConf job = new NutchJob(getConf());
+ conf.set(Generator.GENERATOR_COUNT_MODE,
+ Generator.GENERATOR_COUNT_VALUE_HOST);
+ JobConf job = new NutchJob(getConf());
FileSystem fs = FileSystem.get(job);
- Path dir = new Path(getConf().get("hadoop.tmp.dir"),
- "bench-" + System.currentTimeMillis());
+ Path dir = new Path(getConf().get("hadoop.tmp.dir"), "bench-"
+ + System.currentTimeMillis());
fs.mkdirs(dir);
Path rootUrlDir = new Path(dir, "seed");
fs.mkdirs(rootUrlDir);
@@ -194,7 +207,7 @@
LOG.info("crawl started in: " + dir);
LOG.info("rootUrlDir = " + rootUrlDir);
LOG.info("threads = " + threads);
- LOG.info("depth = " + depth);
+ LOG.info("depth = " + depth);
}
BenchmarkResults res = new BenchmarkResults();
res.delete = delete;
@@ -213,17 +226,17 @@
ParseSegment parseSegment = new ParseSegment(getConf());
CrawlDb crawlDbTool = new CrawlDb(getConf());
LinkDb linkDbTool = new LinkDb(getConf());
-
+
// initialize crawlDb
long start = System.currentTimeMillis();
injector.inject(crawlDb, rootUrlDir);
long delta = System.currentTimeMillis() - start;
res.addTiming("inject", "0", delta);
int i;
- for (i = 0; i < depth; i++) { // generate new segment
+ for (i = 0; i < depth; i++) { // generate new segment
start = System.currentTimeMillis();
- Path[] segs = generator.generate(crawlDb, segments, -1, topN, System
- .currentTimeMillis());
+ Path[] segs = generator.generate(crawlDb, segments, -1, topN,
+ System.currentTimeMillis());
delta = System.currentTimeMillis() - start;
res.addTiming("generate", i + "", delta);
if (segs == null) {
@@ -231,12 +244,12 @@
break;
}
start = System.currentTimeMillis();
- fetcher.fetch(segs[0], threads); // fetch it
+ fetcher.fetch(segs[0], threads); // fetch it
delta = System.currentTimeMillis() - start;
res.addTiming("fetch", i + "", delta);
if (!Fetcher.isParsing(job)) {
start = System.currentTimeMillis();
- parseSegment.parse(segs[0]); // parse it, if needed
+ parseSegment.parse(segs[0]); // parse it, if needed
delta = System.currentTimeMillis() - start;
res.addTiming("parse", i + "", delta);
}
@@ -258,7 +271,9 @@
if (i == 0) {
LOG.warn("No URLs to fetch - check your seed list and URL filters.");
}
- if (LOG.isInfoEnabled()) { LOG.info("crawl finished: " + dir); }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("crawl finished: " + dir);
+ }
res.elapsed = System.currentTimeMillis() - res.elapsed;
CrawlDbReader dbreader = new CrawlDbReader();
dbreader.processStatJob(crawlDb.toString(), conf, false);
Index: src/java/org/apache/nutch/protocol/RobotRules.java
===================================================================
--- src/java/org/apache/nutch/protocol/RobotRules.java (revision 1188252)
+++ src/java/org/apache/nutch/protocol/RobotRules.java (working copy)
@@ -35,9 +35,8 @@
public long getCrawlDelay();
/**
- * Returns false if the robots.txt file
- * prohibits us from accessing the given url, or
- * true otherwise.
+ * Returns false if the robots.txt file prohibits us
+ * from accessing the given url, or true otherwise.
*/
public boolean isAllowed(URL url);
Index: src/java/org/apache/nutch/protocol/Protocol.java
===================================================================
--- src/java/org/apache/nutch/protocol/Protocol.java (revision 1188252)
+++ src/java/org/apache/nutch/protocol/Protocol.java (working copy)
@@ -25,12 +25,11 @@
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.plugin.Pluggable;
-
-/** A retriever of url content. Implemented by protocol extensions. */
+/** A retriever of url content. Implemented by protocol extensions. */
public interface Protocol extends Pluggable, Configurable {
/** The name of the extension point. */
public final static String X_POINT_ID = Protocol.class.getName();
-
+
/**
* Property name. If in the current configuration this property is set to
* true, protocol implementations should handle "politeness" limits
@@ -49,14 +48,18 @@
*/
public final static String CHECK_ROBOTS = "protocol.plugin.check.robots";
- /** Returns the {@link Content} for a fetchlist entry.
+ /**
+ * Returns the {@link Content} for a fetchlist entry.
*/
ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum);
/**
* Retrieve robot rules applicable for this url.
- * @param url url to check
- * @param datum page datum
+ *
+ * @param url
+ * url to check
+ * @param datum
+ * page datum
* @return robot rules (specific for this url or default), never null
*/
RobotRules getRobotRules(Text url, CrawlDatum datum);
Index: src/java/org/apache/nutch/protocol/ProtocolOutput.java
===================================================================
--- src/java/org/apache/nutch/protocol/ProtocolOutput.java (revision 1188252)
+++ src/java/org/apache/nutch/protocol/ProtocolOutput.java (working copy)
@@ -18,8 +18,9 @@
package org.apache.nutch.protocol;
/**
- * Simple aggregate to pass from protocol plugins both content and
- * protocol status.
+ * Simple aggregate to pass from protocol plugins both content and protocol
+ * status.
+ *
* @author Andrzej Bialecki <ab@getopt.org>
*/
public class ProtocolOutput {
@@ -30,12 +31,12 @@
this.content = content;
this.status = status;
}
-
+
public ProtocolOutput(Content content) {
this.content = content;
this.status = ProtocolStatus.STATUS_SUCCESS;
}
-
+
public Content getContent() {
return content;
}
Index: src/java/org/apache/nutch/protocol/ProtocolStatus.java
===================================================================
--- src/java/org/apache/nutch/protocol/ProtocolStatus.java (revision 1188252)
+++ src/java/org/apache/nutch/protocol/ProtocolStatus.java (working copy)
@@ -30,65 +30,76 @@
* @author Andrzej Bialecki
*/
public class ProtocolStatus implements Writable {
-
+
private final static byte VERSION = 2;
-
+
/** Content was retrieved without errors. */
- public static final int SUCCESS = 1;
+ public static final int SUCCESS = 1;
/** Content was not retrieved. Any further errors may be indicated in args. */
- public static final int FAILED = 2;
-
- /** This protocol was not found. Application may attempt to retry later. */
- public static final int PROTO_NOT_FOUND = 10;
+ public static final int FAILED = 2;
+
+ /** This protocol was not found. Application may attempt to retry later. */
+ public static final int PROTO_NOT_FOUND = 10;
/** Resource is gone. */
- public static final int GONE = 11;
+ public static final int GONE = 11;
/** Resource has moved permanently. New url should be found in args. */
- public static final int MOVED = 12;
+ public static final int MOVED = 12;
/** Resource has moved temporarily. New url should be found in args. */
- public static final int TEMP_MOVED = 13;
+ public static final int TEMP_MOVED = 13;
/** Resource was not found. */
- public static final int NOTFOUND = 14;
+ public static final int NOTFOUND = 14;
/** Temporary failure. Application may retry immediately. */
- public static final int RETRY = 15;
- /** Unspecified exception occured. Further information may be provided in args. */
- public static final int EXCEPTION = 16;
+ public static final int RETRY = 15;
+ /**
+ * Unspecified exception occured. Further information may be provided in args.
+ */
+ public static final int EXCEPTION = 16;
/** Access denied - authorization required, but missing/incorrect. */
- public static final int ACCESS_DENIED = 17;
+ public static final int ACCESS_DENIED = 17;
/** Access denied by robots.txt rules. */
- public static final int ROBOTS_DENIED = 18;
+ public static final int ROBOTS_DENIED = 18;
/** Too many redirects. */
- public static final int REDIR_EXCEEDED = 19;
+ public static final int REDIR_EXCEEDED = 19;
/** Not fetching. */
- public static final int NOTFETCHING = 20;
+ public static final int NOTFETCHING = 20;
/** Unchanged since the last fetch. */
- public static final int NOTMODIFIED = 21;
- /** Request was refused by protocol plugins, because it would block.
- * The expected number of milliseconds to wait before retry may be provided
- * in args. */
- public static final int WOULDBLOCK = 22;
+ public static final int NOTMODIFIED = 21;
+ /**
+ * Request was refused by protocol plugins, because it would block. The
+ * expected number of milliseconds to wait before retry may be provided in
+ * args.
+ */
+ public static final int WOULDBLOCK = 22;
/** Thread was blocked http.max.delays times during fetching. */
- public static final int BLOCKED = 23;
-
+ public static final int BLOCKED = 23;
+
// Useful static instances for status codes that don't usually require any
// additional arguments.
- public static final ProtocolStatus STATUS_SUCCESS = new ProtocolStatus(SUCCESS);
+ public static final ProtocolStatus STATUS_SUCCESS = new ProtocolStatus(
+ SUCCESS);
public static final ProtocolStatus STATUS_FAILED = new ProtocolStatus(FAILED);
public static final ProtocolStatus STATUS_GONE = new ProtocolStatus(GONE);
- public static final ProtocolStatus STATUS_NOTFOUND = new ProtocolStatus(NOTFOUND);
+ public static final ProtocolStatus STATUS_NOTFOUND = new ProtocolStatus(
+ NOTFOUND);
public static final ProtocolStatus STATUS_RETRY = new ProtocolStatus(RETRY);
- public static final ProtocolStatus STATUS_ROBOTS_DENIED = new ProtocolStatus(ROBOTS_DENIED);
- public static final ProtocolStatus STATUS_REDIR_EXCEEDED = new ProtocolStatus(REDIR_EXCEEDED);
- public static final ProtocolStatus STATUS_NOTFETCHING = new ProtocolStatus(NOTFETCHING);
- public static final ProtocolStatus STATUS_NOTMODIFIED = new ProtocolStatus(NOTMODIFIED);
- public static final ProtocolStatus STATUS_WOULDBLOCK = new ProtocolStatus(WOULDBLOCK);
- public static final ProtocolStatus STATUS_BLOCKED = new ProtocolStatus(BLOCKED);
-
+ public static final ProtocolStatus STATUS_ROBOTS_DENIED = new ProtocolStatus(
+ ROBOTS_DENIED);
+ public static final ProtocolStatus STATUS_REDIR_EXCEEDED = new ProtocolStatus(
+ REDIR_EXCEEDED);
+ public static final ProtocolStatus STATUS_NOTFETCHING = new ProtocolStatus(
+ NOTFETCHING);
+ public static final ProtocolStatus STATUS_NOTMODIFIED = new ProtocolStatus(
+ NOTMODIFIED);
+ public static final ProtocolStatus STATUS_WOULDBLOCK = new ProtocolStatus(
+ WOULDBLOCK);
+ public static final ProtocolStatus STATUS_BLOCKED = new ProtocolStatus(
+ BLOCKED);
+
private int code;
private long lastModified;
private String[] args;
-
- private static final HashMap codeToName =
- new HashMap();
+
+ private static final HashMap codeToName = new HashMap();
static {
codeToName.put(new Integer(SUCCESS), "success");
codeToName.put(new Integer(FAILED), "failed");
@@ -107,40 +118,41 @@
codeToName.put(new Integer(WOULDBLOCK), "wouldblock");
codeToName.put(new Integer(BLOCKED), "blocked");
}
-
+
public ProtocolStatus() {
-
+
}
public ProtocolStatus(int code, String[] args) {
this.code = code;
this.args = args;
}
-
+
public ProtocolStatus(int code, String[] args, long lastModified) {
this.code = code;
this.args = args;
this.lastModified = lastModified;
}
-
+
public ProtocolStatus(int code) {
this(code, null);
}
-
+
public ProtocolStatus(int code, long lastModified) {
this(code, null, lastModified);
}
-
+
public ProtocolStatus(int code, Object message) {
this(code, message, 0L);
}
-
+
public ProtocolStatus(int code, Object message, long lastModified) {
this.code = code;
this.lastModified = lastModified;
- if (message != null) this.args = new String[]{String.valueOf(message)};
+ if (message != null)
+ this.args = new String[] { String.valueOf(message) };
}
-
+
public ProtocolStatus(Throwable t) {
this(EXCEPTION, t);
}
@@ -150,10 +162,10 @@
res.readFields(in);
return res;
}
-
+
public void readFields(DataInput in) throws IOException {
byte version = in.readByte();
- switch(version) {
+ switch (version) {
case 1:
code = in.readByte();
lastModified = in.readLong();
@@ -168,10 +180,10 @@
throw new VersionMismatchException(VERSION, version);
}
}
-
+
public void write(DataOutput out) throws IOException {
out.writeByte(VERSION);
- out.writeByte((byte)code);
+ out.writeByte((byte) code);
out.writeLong(lastModified);
if (args == null) {
out.writeInt(-1);
@@ -183,7 +195,7 @@
public void setArgs(String[] args) {
this.args = args;
}
-
+
public String[] getArgs() {
return args;
}
@@ -195,74 +207,77 @@
public String getName() {
return codeToName.get(this.code);
}
-
+
public void setCode(int code) {
this.code = code;
}
-
+
public boolean isSuccess() {
- return code == SUCCESS;
+ return code == SUCCESS;
}
-
+
public boolean isTransientFailure() {
- return
- code == ACCESS_DENIED ||
- code == EXCEPTION ||
- code == REDIR_EXCEEDED ||
- code == RETRY ||
- code == TEMP_MOVED ||
- code == WOULDBLOCK ||
- code == PROTO_NOT_FOUND;
+ return code == ACCESS_DENIED || code == EXCEPTION || code == REDIR_EXCEEDED
+ || code == RETRY || code == TEMP_MOVED || code == WOULDBLOCK
+ || code == PROTO_NOT_FOUND;
}
-
+
public boolean isPermanentFailure() {
- return
- code == FAILED ||
- code == GONE ||
- code == MOVED ||
- code == NOTFOUND ||
- code == ROBOTS_DENIED;
+ return code == FAILED || code == GONE || code == MOVED || code == NOTFOUND
+ || code == ROBOTS_DENIED;
}
-
+
public String getMessage() {
- if (args != null && args.length > 0) return args[0];
+ if (args != null && args.length > 0)
+ return args[0];
return null;
}
-
+
public void setMessage(String msg) {
- if (args != null && args.length > 0) args[0] = msg;
- else args = new String[] {msg};
+ if (args != null && args.length > 0)
+ args[0] = msg;
+ else
+ args = new String[] { msg };
}
-
+
public long getLastModified() {
return lastModified;
}
-
+
public void setLastModified(long lastModified) {
this.lastModified = lastModified;
}
-
+
public boolean equals(Object o) {
- if (o == null) return false;
- if (!(o instanceof ProtocolStatus)) return false;
- ProtocolStatus other = (ProtocolStatus)o;
- if (this.code != other.code || this.lastModified != other.lastModified) return false;
+ if (o == null)
+ return false;
+ if (!(o instanceof ProtocolStatus))
+ return false;
+ ProtocolStatus other = (ProtocolStatus) o;
+ if (this.code != other.code || this.lastModified != other.lastModified)
+ return false;
if (this.args == null) {
- if (other.args == null) return true;
- else return false;
+ if (other.args == null)
+ return true;
+ else
+ return false;
} else {
- if (other.args == null) return false;
- if (other.args.length != this.args.length) return false;
+ if (other.args == null)
+ return false;
+ if (other.args.length != this.args.length)
+ return false;
for (int i = 0; i < this.args.length; i++) {
- if (!this.args[i].equals(other.args[i])) return false;
+ if (!this.args[i].equals(other.args[i]))
+ return false;
}
}
return true;
}
-
+
public String toString() {
StringBuffer res = new StringBuffer();
- res.append(codeToName.get(new Integer(code)) + "(" + code + "), lastModified=" + lastModified);
+ res.append(codeToName.get(new Integer(code)) + "(" + code
+ + "), lastModified=" + lastModified);
if (args != null) {
if (args.length == 1) {
res.append(": " + String.valueOf(args[0]));
Index: src/java/org/apache/nutch/protocol/ProtocolFactory.java
===================================================================
--- src/java/org/apache/nutch/protocol/ProtocolFactory.java (revision 1188252)
+++ src/java/org/apache/nutch/protocol/ProtocolFactory.java (working copy)
@@ -38,7 +38,8 @@
*/
public class ProtocolFactory {
- public static final Logger LOG = LoggerFactory.getLogger(ProtocolFactory.class);
+ public static final Logger LOG = LoggerFactory
+ .getLogger(ProtocolFactory.class);
private ExtensionPoint extensionPoint;
@@ -59,7 +60,8 @@
*
* @param urlString
* Url String
- * @return The appropriate {@link Protocol} implementation for a given {@link URL}.
+ * @return The appropriate {@link Protocol} implementation for a given
+ * {@link URL}.
* @throws ProtocolNotFound
* when Protocol can not be found for urlString
*/
@@ -106,13 +108,14 @@
}
return null;
}
-
- boolean contains(String what, String where){
- String parts[]=where.split("[, ]");
- for(int i=0;i
* http://www.iana.org/assignments/media-types/
*/
@@ -259,13 +262,12 @@
}
Options opts = new Options();
Configuration conf = NutchConfiguration.create();
-
- GenericOptionsParser parser =
- new GenericOptionsParser(conf, opts, argv);
-
+
+ GenericOptionsParser parser = new GenericOptionsParser(conf, opts, argv);
+
String[] remainingArgs = parser.getRemainingArgs();
FileSystem fs = FileSystem.get(conf);
-
+
try {
int recno = Integer.parseInt(remainingArgs[0]);
String segment = remainingArgs[1];
Index: src/java/org/apache/nutch/protocol/ProtocolNotFound.java
===================================================================
--- src/java/org/apache/nutch/protocol/ProtocolNotFound.java (revision 1188252)
+++ src/java/org/apache/nutch/protocol/ProtocolNotFound.java (working copy)
@@ -22,7 +22,7 @@
private String url;
public ProtocolNotFound(String url) {
- this(url, "protocol not found for url="+url);
+ this(url, "protocol not found for url=" + url);
}
public ProtocolNotFound(String url, String message) {
@@ -30,5 +30,7 @@
this.url = url;
}
- public String getUrl() { return url; }
+ public String getUrl() {
+ return url;
+ }
}
Index: src/java/org/apache/nutch/segment/SegmentMerger.java
===================================================================
--- src/java/org/apache/nutch/segment/SegmentMerger.java (revision 1188252)
+++ src/java/org/apache/nutch/segment/SegmentMerger.java (working copy)
@@ -72,40 +72,47 @@
*
* Also, it's possible to slice the resulting segment into chunks of fixed size.
*
- * Important Notes
- * Which parts are merged?
- * It doesn't make sense to merge data from segments, which are at different stages
- * of processing (e.g. one unfetched segment, one fetched but not parsed, and
- * one fetched and parsed). Therefore, prior to merging, the tool will determine
- * the lowest common set of input data, and only this data will be merged.
- * This may have some unintended consequences:
- * e.g. if majority of input segments are fetched and parsed, but one of them is unfetched,
- * the tool will fall back to just merging fetchlists, and it will skip all other data
- * from all segments.
+ * Important Notes
Which parts are merged?
+ *
+ * It doesn't make sense to merge data from segments, which are at different
+ * stages of processing (e.g. one unfetched segment, one fetched but not parsed,
+ * and one fetched and parsed). Therefore, prior to merging, the tool will
+ * determine the lowest common set of input data, and only this data will be
+ * merged. This may have some unintended consequences: e.g. if majority of input
+ * segments are fetched and parsed, but one of them is unfetched, the tool will
+ * fall back to just merging fetchlists, and it will skip all other data from
+ * all segments.
+ *
* Merging fetchlists
- * Merging segments, which contain just fetchlists (i.e. prior to fetching)
- * is not recommended, because this tool (unlike the {@link org.apache.nutch.crawl.Generator}
- * doesn't ensure that fetchlist parts for each map task are disjoint.
*
+ * Merging segments, which contain just fetchlists (i.e. prior to fetching) is
+ * not recommended, because this tool (unlike the
+ * {@link org.apache.nutch.crawl.Generator} doesn't ensure that fetchlist parts
+ * for each map task are disjoint.
+ *
+ *
*
Duplicate content
- * Merging segments removes older content whenever possible (see below). However,
- * this is NOT the same as de-duplication, which in addition removes identical
- * content found at different URL-s. In other words, running DeleteDuplicates is
- * still necessary.
+ * Merging segments removes older content whenever possible (see below).
+ * However, this is NOT the same as de-duplication, which in addition removes
+ * identical content found at different URL-s. In other words, running
+ * DeleteDuplicates is still necessary.
*
- * For some types of data (especially ParseText) it's not possible to determine
- * which version is really older. Therefore the tool always uses segment names as
- * timestamps, for all types of input data. Segment names are compared in forward lexicographic
- * order (0-9a-zA-Z), and data from segments with "higher" names will prevail.
- * It follows then that it is extremely important that segments be named in an
- * increasing lexicographic order as their creation time increases.
*
+ * For some types of data (especially ParseText) it's not possible to determine
+ * which version is really older. Therefore the tool always uses segment names
+ * as timestamps, for all types of input data. Segment names are compared in
+ * forward lexicographic order (0-9a-zA-Z), and data from segments with "higher"
+ * names will prevail. It follows then that it is extremely important that
+ * segments be named in an increasing lexicographic order as their creation time
+ * increases.
+ *
+ *
*
Merging and indexes
* Merged segment gets a different name. Since Indexer embeds segment names in
- * indexes, any indexes originally created for the input segments will NOT work with the
- * merged segment. Newly created merged segment(s) need to be indexed afresh.
- * This tool doesn't use existing indexes in any way, so if
- * you plan to merge segments you don't have to index them prior to merging.
+ * indexes, any indexes originally created for the input segments will NOT work
+ * with the merged segment. Newly created merged segment(s) need to be indexed
+ * afresh. This tool doesn't use existing indexes in any way, so if you plan to
+ * merge segments you don't have to index them prior to merging.
*
*
* @author Andrzej Bialecki
@@ -113,7 +120,8 @@
public class SegmentMerger extends Configured implements
Mapper,
Reducer {
- private static final Logger LOG = LoggerFactory.getLogger(SegmentMerger.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SegmentMerger.class);
private static final String SEGMENT_PART_KEY = "part";
private static final String SEGMENT_SLICE_KEY = "slice";
@@ -123,20 +131,21 @@
private SegmentMergeFilters mergeFilters = null;
private long sliceSize = -1;
private long curCount = 0;
-
+
/**
- * Wraps inputs in an {@link MetaWrapper}, to permit merging different
- * types in reduce and use additional metadata.
+ * Wraps inputs in an {@link MetaWrapper}, to permit merging different types
+ * in reduce and use additional metadata.
*/
public static class ObjectInputFormat extends
- SequenceFileInputFormat {
-
+ SequenceFileInputFormat {
+
@Override
- public RecordReader getRecordReader(final InputSplit split,
- final JobConf job, Reporter reporter) throws IOException {
+ public RecordReader getRecordReader(
+ final InputSplit split, final JobConf job, Reporter reporter)
+ throws IOException {
reporter.setStatus(split.toString());
-
+
// find part name
SegmentPart segmentPart;
final String spString;
@@ -147,10 +156,10 @@
} catch (IOException e) {
throw new RuntimeException("Cannot identify segment:", e);
}
-
- SequenceFile.Reader reader =
- new SequenceFile.Reader(FileSystem.get(job), fSplit.getPath(), job);
-
+
+ SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(job),
+ fSplit.getPath(), job);
+
final Writable w;
try {
w = (Writable) reader.getValueClass().newInstance();
@@ -163,13 +172,14 @@
// ignore
}
}
- final SequenceFileRecordReader splitReader =
- new SequenceFileRecordReader(job, (FileSplit)split);
+ final SequenceFileRecordReader splitReader = new SequenceFileRecordReader(
+ job, (FileSplit) split);
try {
return new SequenceFileRecordReader(job, fSplit) {
-
- public synchronized boolean next(Text key, MetaWrapper wrapper) throws IOException {
+
+ public synchronized boolean next(Text key, MetaWrapper wrapper)
+ throws IOException {
LOG.debug("Running OIF.next()");
boolean res = splitReader.next(key, w);
@@ -177,17 +187,17 @@
wrapper.setMeta(SEGMENT_PART_KEY, spString);
return res;
}
-
+
@Override
public synchronized void close() throws IOException {
splitReader.close();
}
-
+
@Override
public MetaWrapper createValue() {
return new MetaWrapper();
}
-
+
};
} catch (IOException e) {
throw new RuntimeException("Cannot create RecordReader: ", e);
@@ -195,11 +205,14 @@
}
}
- public static class SegmentOutputFormat extends FileOutputFormat {
+ public static class SegmentOutputFormat extends
+ FileOutputFormat {
private static final String DEFAULT_SLICE = "default";
-
+
@Override
- public RecordWriter getRecordWriter(final FileSystem fs, final JobConf job, final String name, final Progressable progress) throws IOException {
+ public RecordWriter getRecordWriter(final FileSystem fs,
+ final JobConf job, final String name, final Progressable progress)
+ throws IOException {
return new RecordWriter() {
MapFile.Writer c_out = null;
MapFile.Writer f_out = null;
@@ -209,24 +222,26 @@
SequenceFile.Writer p_out = null;
HashMap sliceWriters = new HashMap();
String segmentName = job.get("segment.merger.segmentName");
-
+
public void write(Text key, MetaWrapper wrapper) throws IOException {
// unwrap
SegmentPart sp = SegmentPart.parse(wrapper.getMeta(SEGMENT_PART_KEY));
- Writable o = (Writable)wrapper.get();
+ Writable o = (Writable) wrapper.get();
String slice = wrapper.getMeta(SEGMENT_SLICE_KEY);
if (o instanceof CrawlDatum) {
if (sp.partName.equals(CrawlDatum.GENERATE_DIR_NAME)) {
g_out = ensureSequenceFile(slice, CrawlDatum.GENERATE_DIR_NAME);
g_out.append(key, o);
} else if (sp.partName.equals(CrawlDatum.FETCH_DIR_NAME)) {
- f_out = ensureMapFile(slice, CrawlDatum.FETCH_DIR_NAME, CrawlDatum.class);
+ f_out = ensureMapFile(slice, CrawlDatum.FETCH_DIR_NAME,
+ CrawlDatum.class);
f_out.append(key, o);
} else if (sp.partName.equals(CrawlDatum.PARSE_DIR_NAME)) {
p_out = ensureSequenceFile(slice, CrawlDatum.PARSE_DIR_NAME);
p_out.append(key, o);
} else {
- throw new IOException("Cannot determine segment part: " + sp.partName);
+ throw new IOException("Cannot determine segment part: "
+ + sp.partName);
}
} else if (o instanceof Content) {
c_out = ensureMapFile(slice, Content.DIR_NAME, Content.class);
@@ -234,9 +249,11 @@
} else if (o instanceof ParseData) {
// update the segment name inside contentMeta - required by Indexer
if (slice == null) {
- ((ParseData)o).getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName);
+ ((ParseData) o).getContentMeta().set(Nutch.SEGMENT_NAME_KEY,
+ segmentName);
} else {
- ((ParseData)o).getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName + "-" + slice);
+ ((ParseData) o).getContentMeta().set(Nutch.SEGMENT_NAME_KEY,
+ segmentName + "-" + slice);
}
pd_out = ensureMapFile(slice, ParseData.DIR_NAME, ParseData.class);
pd_out.append(key, o);
@@ -245,20 +262,26 @@
pt_out.append(key, o);
}
}
-
+
// lazily create SequenceFile-s.
- private SequenceFile.Writer ensureSequenceFile(String slice, String dirName) throws IOException {
- if (slice == null) slice = DEFAULT_SLICE;
- SequenceFile.Writer res = (SequenceFile.Writer)sliceWriters.get(slice + dirName);
- if (res != null) return res;
+ private SequenceFile.Writer ensureSequenceFile(String slice,
+ String dirName) throws IOException {
+ if (slice == null)
+ slice = DEFAULT_SLICE;
+ SequenceFile.Writer res = (SequenceFile.Writer) sliceWriters
+ .get(slice + dirName);
+ if (res != null)
+ return res;
Path wname;
Path out = FileOutputFormat.getOutputPath(job);
if (slice == DEFAULT_SLICE) {
- wname = new Path(new Path(new Path(out, segmentName), dirName), name);
+ wname = new Path(new Path(new Path(out, segmentName), dirName),
+ name);
} else {
- wname = new Path(new Path(new Path(out, segmentName + "-" + slice), dirName), name);
+ wname = new Path(new Path(new Path(out, segmentName + "-" + slice),
+ dirName), name);
}
- res = SequenceFile.createWriter(fs, job, wname, Text.class,
+ res = SequenceFile.createWriter(fs, job, wname, Text.class,
CrawlDatum.class,
SequenceFileOutputFormat.getOutputCompressionType(job), progress);
sliceWriters.put(slice + dirName, res);
@@ -266,23 +289,30 @@
}
// lazily create MapFile-s.
- private MapFile.Writer ensureMapFile(String slice, String dirName, Class clazz) throws IOException {
- if (slice == null) slice = DEFAULT_SLICE;
- MapFile.Writer res = (MapFile.Writer)sliceWriters.get(slice + dirName);
- if (res != null) return res;
+ private MapFile.Writer ensureMapFile(String slice, String dirName,
+ Class clazz) throws IOException {
+ if (slice == null)
+ slice = DEFAULT_SLICE;
+ MapFile.Writer res = (MapFile.Writer) sliceWriters.get(slice
+ + dirName);
+ if (res != null)
+ return res;
Path wname;
Path out = FileOutputFormat.getOutputPath(job);
if (slice == DEFAULT_SLICE) {
- wname = new Path(new Path(new Path(out, segmentName), dirName), name);
+ wname = new Path(new Path(new Path(out, segmentName), dirName),
+ name);
} else {
- wname = new Path(new Path(new Path(out, segmentName + "-" + slice), dirName), name);
+ wname = new Path(new Path(new Path(out, segmentName + "-" + slice),
+ dirName), name);
}
- CompressionType compType =
- SequenceFileOutputFormat.getOutputCompressionType(job);
+ CompressionType compType = SequenceFileOutputFormat
+ .getOutputCompressionType(job);
if (clazz.isAssignableFrom(ParseText.class)) {
compType = CompressionType.RECORD;
}
- res = new MapFile.Writer(job, fs, wname.toString(), Text.class, clazz, compType, progress);
+ res = new MapFile.Writer(job, fs, wname.toString(), Text.class,
+ clazz, compType, progress);
sliceWriters.put(slice + dirName, res);
return res;
}
@@ -292,9 +322,9 @@
while (it.hasNext()) {
Object o = it.next();
if (o instanceof SequenceFile.Writer) {
- ((SequenceFile.Writer)o).close();
+ ((SequenceFile.Writer) o).close();
} else {
- ((MapFile.Writer)o).close();
+ ((MapFile.Writer) o).close();
}
}
}
@@ -305,14 +335,15 @@
public SegmentMerger() {
super(null);
}
-
+
public SegmentMerger(Configuration conf) {
super(conf);
}
-
+
public void setConf(Configuration conf) {
super.setConf(conf);
- if (conf == null) return;
+ if (conf == null)
+ return;
if (conf.getBoolean("segment.merger.filter", false)) {
filters = new URLFilters(conf);
mergeFilters = new SegmentMergeFilters(conf);
@@ -334,15 +365,18 @@
sliceSize = sliceSize / conf.getNumReduceTasks();
}
}
-
+
private Text newKey = new Text();
-
+
public void map(Text key, MetaWrapper value,
- OutputCollector output, Reporter reporter) throws IOException {
+ OutputCollector output, Reporter reporter)
+ throws IOException {
String url = key.toString();
if (normalizers != null) {
try {
- url = normalizers.normalize(url, URLNormalizers.SCOPE_DEFAULT); // normalize the url
+ url = normalizers.normalize(url, URLNormalizers.SCOPE_DEFAULT); // normalize
+ // the
+ // url
} catch (Exception e) {
LOG.warn("Skipping " + url + ":" + e.getMessage());
url = null;
@@ -356,7 +390,7 @@
url = null;
}
}
- if(url != null) {
+ if (url != null) {
newKey.set(url);
output.collect(newKey, value);
}
@@ -364,12 +398,13 @@
/**
* NOTE: in selecting the latest version we rely exclusively on the segment
- * name (not all segment data contain time information). Therefore it is extremely
- * important that segments be named in an increasing lexicographic order as
- * their creation time increases.
+ * name (not all segment data contain time information). Therefore it is
+ * extremely important that segments be named in an increasing lexicographic
+ * order as their creation time increases.
*/
public void reduce(Text key, Iterator values,
- OutputCollector output, Reporter reporter) throws IOException {
+ OutputCollector output, Reporter reporter)
+ throws IOException {
CrawlDatum lastG = null;
CrawlDatum lastF = null;
CrawlDatum lastSig = null;
@@ -382,18 +417,17 @@
String lastCname = null;
String lastPDname = null;
String lastPTname = null;
- TreeMap> linked =
- new TreeMap>();
+ TreeMap> linked = new TreeMap>();
while (values.hasNext()) {
MetaWrapper wrapper = values.next();
Object o = wrapper.get();
String spString = wrapper.getMeta(SEGMENT_PART_KEY);
if (spString == null) {
- throw new IOException("Null segment part, key=" + key);
+ throw new IOException("Null segment part, key=" + key);
}
SegmentPart sp = SegmentPart.parse(spString);
if (o instanceof CrawlDatum) {
- CrawlDatum val = (CrawlDatum)o;
+ CrawlDatum val = (CrawlDatum) o;
// check which output dir it belongs to
if (sp.partName.equals(CrawlDatum.GENERATE_DIR_NAME)) {
if (lastG == null) {
@@ -443,43 +477,43 @@
}
} else if (o instanceof Content) {
if (lastC == null) {
- lastC = (Content)o;
+ lastC = (Content) o;
lastCname = sp.segmentName;
} else {
if (lastCname.compareTo(sp.segmentName) < 0) {
- lastC = (Content)o;
+ lastC = (Content) o;
lastCname = sp.segmentName;
}
}
} else if (o instanceof ParseData) {
if (lastPD == null) {
- lastPD = (ParseData)o;
+ lastPD = (ParseData) o;
lastPDname = sp.segmentName;
} else {
if (lastPDname.compareTo(sp.segmentName) < 0) {
- lastPD = (ParseData)o;
+ lastPD = (ParseData) o;
lastPDname = sp.segmentName;
}
}
} else if (o instanceof ParseText) {
if (lastPT == null) {
- lastPT = (ParseText)o;
+ lastPT = (ParseText) o;
lastPTname = sp.segmentName;
} else {
if (lastPTname.compareTo(sp.segmentName) < 0) {
- lastPT = (ParseText)o;
+ lastPT = (ParseText) o;
lastPTname = sp.segmentName;
}
}
}
}
- // perform filtering based on full merge record
- if (mergeFilters != null &&
- !mergeFilters.filter(key, lastG, lastF, lastSig, lastC, lastPD, lastPT,
- linked.isEmpty() ? null : linked.lastEntry().getValue())){
+ // perform filtering based on full merge record
+ if (mergeFilters != null
+ && !mergeFilters.filter(key, lastG, lastF, lastSig, lastC, lastPD,
+ lastPT, linked.isEmpty() ? null : linked.lastEntry().getValue())) {
return;
}
-
+
curCount++;
String sliceName = null;
MetaWrapper wrapper = new MetaWrapper();
@@ -545,10 +579,12 @@
}
}
- public void merge(Path out, Path[] segs, boolean filter, boolean normalize, long slice) throws Exception {
+ public void merge(Path out, Path[] segs, boolean filter, boolean normalize,
+ long slice) throws Exception {
String segmentName = Generator.generateSegmentName();
if (LOG.isInfoEnabled()) {
- LOG.info("Merging " + segs.length + " segments to " + out + "/" + segmentName);
+ LOG.info("Merging " + segs.length + " segments to " + out + "/"
+ + segmentName);
}
JobConf job = new NutchJob(getConf());
job.setJobName("mergesegs " + out + "/" + segmentName);
@@ -589,17 +625,24 @@
pt = pt && fs.exists(ptDir);
}
StringBuffer sb = new StringBuffer();
- if (c) sb.append(" " + Content.DIR_NAME);
- if (g) sb.append(" " + CrawlDatum.GENERATE_DIR_NAME);
- if (f) sb.append(" " + CrawlDatum.FETCH_DIR_NAME);
- if (p) sb.append(" " + CrawlDatum.PARSE_DIR_NAME);
- if (pd) sb.append(" " + ParseData.DIR_NAME);
- if (pt) sb.append(" " + ParseText.DIR_NAME);
+ if (c)
+ sb.append(" " + Content.DIR_NAME);
+ if (g)
+ sb.append(" " + CrawlDatum.GENERATE_DIR_NAME);
+ if (f)
+ sb.append(" " + CrawlDatum.FETCH_DIR_NAME);
+ if (p)
+ sb.append(" " + CrawlDatum.PARSE_DIR_NAME);
+ if (pd)
+ sb.append(" " + ParseData.DIR_NAME);
+ if (pt)
+ sb.append(" " + ParseText.DIR_NAME);
if (LOG.isInfoEnabled()) {
LOG.info("SegmentMerger: using segment data from:" + sb.toString());
}
for (int i = 0; i < segs.length; i++) {
- if (segs[i] == null) continue;
+ if (segs[i] == null)
+ continue;
if (g) {
Path gDir = new Path(segs[i], CrawlDatum.GENERATE_DIR_NAME);
FileInputFormat.addInputPath(job, gDir);
@@ -632,9 +675,9 @@
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(MetaWrapper.class);
job.setOutputFormat(SegmentOutputFormat.class);
-
+
setConf(job);
-
+
JobClient.runJob(job);
}
@@ -643,12 +686,17 @@
*/
public static void main(String[] args) throws Exception {
if (args.length < 2) {
- System.err.println("SegmentMerger output_dir (-dir segments | seg1 seg2 ...) [-filter] [-slice NNNN]");
- System.err.println("\toutput_dir\tname of the parent dir for output segment slice(s)");
- System.err.println("\t-dir segments\tparent dir containing several segments");
+ System.err
+ .println("SegmentMerger output_dir (-dir segments | seg1 seg2 ...) [-filter] [-slice NNNN]");
+ System.err
+ .println("\toutput_dir\tname of the parent dir for output segment slice(s)");
+ System.err
+ .println("\t-dir segments\tparent dir containing several segments");
System.err.println("\tseg1 seg2 ...\tlist of segment dirs");
- System.err.println("\t-filter\t\tfilter out URL-s prohibited by current URLFilters");
- System.err.println("\t-slice NNNN\tcreate many output segments, each containing NNNN URLs");
+ System.err
+ .println("\t-filter\t\tfilter out URL-s prohibited by current URLFilters");
+ System.err
+ .println("\t-slice NNNN\tcreate many output segments, each containing NNNN URLs");
return;
}
Configuration conf = NutchConfiguration.create();
@@ -680,7 +728,8 @@
return;
}
SegmentMerger merger = new SegmentMerger(conf);
- merger.merge(out, segs.toArray(new Path[segs.size()]), filter, normalize, sliceSize);
+ merger.merge(out, segs.toArray(new Path[segs.size()]), filter, normalize,
+ sliceSize);
}
}
Index: src/java/org/apache/nutch/segment/SegmentReader.java
===================================================================
--- src/java/org/apache/nutch/segment/SegmentReader.java (revision 1188252)
+++ src/java/org/apache/nutch/segment/SegmentReader.java (working copy)
@@ -77,7 +77,7 @@
public static final Logger LOG = LoggerFactory.getLogger(SegmentReader.class);
long recNo = 0L;
-
+
private boolean co, fe, ge, pa, pd, pt;
private FileSystem fs;
@@ -86,32 +86,37 @@
private Text newKey = new Text();
public void map(WritableComparable key, Writable value,
- OutputCollector collector, Reporter reporter) throws IOException {
+ OutputCollector collector, Reporter reporter)
+ throws IOException {
// convert on the fly from old formats with UTF8 keys
if (key instanceof UTF8) {
newKey.set(key.toString());
key = newKey;
}
- collector.collect((Text)key, new NutchWritable(value));
+ collector.collect((Text) key, new NutchWritable(value));
}
-
+
}
/** Implements a text output format */
public static class TextOutputFormat extends
FileOutputFormat {
public RecordWriter getRecordWriter(
- final FileSystem fs, JobConf job,
- String name, final Progressable progress) throws IOException {
+ final FileSystem fs, JobConf job, String name,
+ final Progressable progress) throws IOException {
- final Path segmentDumpFile = new Path(FileOutputFormat.getOutputPath(job), name);
+ final Path segmentDumpFile = new Path(
+ FileOutputFormat.getOutputPath(job), name);
// Get the old copy out of the way
- if (fs.exists(segmentDumpFile)) fs.delete(segmentDumpFile, true);
+ if (fs.exists(segmentDumpFile))
+ fs.delete(segmentDumpFile, true);
- final PrintStream printStream = new PrintStream(fs.create(segmentDumpFile));
+ final PrintStream printStream = new PrintStream(
+ fs.create(segmentDumpFile));
return new RecordWriter() {
- public synchronized void write(WritableComparable key, Writable value) throws IOException {
+ public synchronized void write(WritableComparable key, Writable value)
+ throws IOException {
printStream.println(value);
}
@@ -125,9 +130,9 @@
public SegmentReader() {
super(null);
}
-
- public SegmentReader(Configuration conf, boolean co, boolean fe, boolean ge, boolean pa,
- boolean pd, boolean pt) {
+
+ public SegmentReader(Configuration conf, boolean co, boolean fe, boolean ge,
+ boolean pa, boolean pd, boolean pt) {
super(conf);
this.co = co;
this.fe = fe;
@@ -167,12 +172,12 @@
job.setBoolean("segment.reader.pt", this.pt);
return job;
}
-
- public void close() {}
+ public void close() {
+ }
+
public void reduce(Text key, Iterator values,
- OutputCollector output, Reporter reporter)
- throws IOException {
+ OutputCollector output, Reporter reporter) throws IOException {
StringBuffer dump = new StringBuffer();
dump.append("\nRecno:: ").append(recNo++).append("\n");
@@ -195,7 +200,7 @@
}
public void dump(Path segment, Path output) throws IOException {
-
+
if (LOG.isInfoEnabled()) {
LOG.info("SegmentReader: dump segment: " + segment);
}
@@ -203,20 +208,30 @@
JobConf job = createJobConf();
job.setJobName("read " + segment);
- if (ge) FileInputFormat.addInputPath(job, new Path(segment, CrawlDatum.GENERATE_DIR_NAME));
- if (fe) FileInputFormat.addInputPath(job, new Path(segment, CrawlDatum.FETCH_DIR_NAME));
- if (pa) FileInputFormat.addInputPath(job, new Path(segment, CrawlDatum.PARSE_DIR_NAME));
- if (co) FileInputFormat.addInputPath(job, new Path(segment, Content.DIR_NAME));
- if (pd) FileInputFormat.addInputPath(job, new Path(segment, ParseData.DIR_NAME));
- if (pt) FileInputFormat.addInputPath(job, new Path(segment, ParseText.DIR_NAME));
+ if (ge)
+ FileInputFormat.addInputPath(job, new Path(segment,
+ CrawlDatum.GENERATE_DIR_NAME));
+ if (fe)
+ FileInputFormat.addInputPath(job, new Path(segment,
+ CrawlDatum.FETCH_DIR_NAME));
+ if (pa)
+ FileInputFormat.addInputPath(job, new Path(segment,
+ CrawlDatum.PARSE_DIR_NAME));
+ if (co)
+ FileInputFormat.addInputPath(job, new Path(segment, Content.DIR_NAME));
+ if (pd)
+ FileInputFormat.addInputPath(job, new Path(segment, ParseData.DIR_NAME));
+ if (pt)
+ FileInputFormat.addInputPath(job, new Path(segment, ParseText.DIR_NAME));
job.setInputFormat(SequenceFileInputFormat.class);
job.setMapperClass(InputCompatMapper.class);
job.setReducerClass(SegmentReader.class);
- Path tempDir = new Path(job.get("hadoop.tmp.dir", "/tmp") + "/segread-" + new java.util.Random().nextInt());
+ Path tempDir = new Path(job.get("hadoop.tmp.dir", "/tmp") + "/segread-"
+ + new java.util.Random().nextInt());
fs.delete(tempDir, true);
-
+
FileOutputFormat.setOutputPath(job, tempDir);
job.setOutputFormat(TextOutputFormat.class);
job.setOutputKeyClass(Text.class);
@@ -229,22 +244,25 @@
// remove the old file
fs.delete(dumpFile, true);
- FileStatus[] fstats = fs.listStatus(tempDir, HadoopFSUtil.getPassAllFilter());
+ FileStatus[] fstats = fs.listStatus(tempDir,
+ HadoopFSUtil.getPassAllFilter());
Path[] files = HadoopFSUtil.getPaths(fstats);
PrintWriter writer = null;
int currentRecordNumber = 0;
if (files.length > 0) {
- writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(fs.create(dumpFile))));
+ writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(
+ fs.create(dumpFile))));
try {
for (int i = 0; i < files.length; i++) {
Path partFile = (Path) files[i];
try {
- currentRecordNumber = append(fs, job, partFile, writer, currentRecordNumber);
+ currentRecordNumber = append(fs, job, partFile, writer,
+ currentRecordNumber);
} catch (IOException exception) {
if (LOG.isWarnEnabled()) {
- LOG.warn("Couldn't copy the content of " + partFile.toString() +
- " into " + dumpFile.toString());
+ LOG.warn("Couldn't copy the content of " + partFile.toString()
+ + " into " + dumpFile.toString());
LOG.warn(exception.getMessage());
}
}
@@ -254,13 +272,16 @@
}
}
fs.delete(tempDir);
- if (LOG.isInfoEnabled()) { LOG.info("SegmentReader: done"); }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("SegmentReader: done");
+ }
}
/** Appends two files and updates the Recno counter */
- private int append(FileSystem fs, Configuration conf, Path src, PrintWriter writer, int currentRecordNumber)
- throws IOException {
- BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(src)));
+ private int append(FileSystem fs, Configuration conf, Path src,
+ PrintWriter writer, int currentRecordNumber) throws IOException {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(
+ fs.open(src)));
try {
String line = reader.readLine();
while (line != null) {
@@ -277,89 +298,101 @@
}
private static final String[][] keys = new String[][] {
- {"co", "Content::\n"},
- {"ge", "Crawl Generate::\n"},
- {"fe", "Crawl Fetch::\n"},
- {"pa", "Crawl Parse::\n"},
- {"pd", "ParseData::\n"},
- {"pt", "ParseText::\n"}
- };
+ { "co", "Content::\n" }, { "ge", "Crawl Generate::\n" },
+ { "fe", "Crawl Fetch::\n" }, { "pa", "Crawl Parse::\n" },
+ { "pd", "ParseData::\n" }, { "pt", "ParseText::\n" } };
public void get(final Path segment, final Text key, Writer writer,
- final Map> results) throws Exception {
+ final Map> results) throws Exception {
LOG.info("SegmentReader: get '" + key + "'");
ArrayList threads = new ArrayList();
- if (co) threads.add(new Thread() {
- public void run() {
- try {
- List res = getMapRecords(new Path(segment, Content.DIR_NAME), key);
- results.put("co", res);
- } catch (Exception e) {
- e.printStackTrace(LogUtil.getWarnStream(LOG));
+ if (co)
+ threads.add(new Thread() {
+ public void run() {
+ try {
+ List res = getMapRecords(new Path(segment,
+ Content.DIR_NAME), key);
+ results.put("co", res);
+ } catch (Exception e) {
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ }
}
- }
- });
- if (fe) threads.add(new Thread() {
- public void run() {
- try {
- List res = getMapRecords(new Path(segment, CrawlDatum.FETCH_DIR_NAME), key);
- results.put("fe", res);
- } catch (Exception e) {
- e.printStackTrace(LogUtil.getWarnStream(LOG));
+ });
+ if (fe)
+ threads.add(new Thread() {
+ public void run() {
+ try {
+ List res = getMapRecords(new Path(segment,
+ CrawlDatum.FETCH_DIR_NAME), key);
+ results.put("fe", res);
+ } catch (Exception e) {
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ }
}
- }
- });
- if (ge) threads.add(new Thread() {
- public void run() {
- try {
- List res = getSeqRecords(new Path(segment, CrawlDatum.GENERATE_DIR_NAME), key);
- results.put("ge", res);
- } catch (Exception e) {
- e.printStackTrace(LogUtil.getWarnStream(LOG));
+ });
+ if (ge)
+ threads.add(new Thread() {
+ public void run() {
+ try {
+ List res = getSeqRecords(new Path(segment,
+ CrawlDatum.GENERATE_DIR_NAME), key);
+ results.put("ge", res);
+ } catch (Exception e) {
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ }
}
- }
- });
- if (pa) threads.add(new Thread() {
- public void run() {
- try {
- List res = getSeqRecords(new Path(segment, CrawlDatum.PARSE_DIR_NAME), key);
- results.put("pa", res);
- } catch (Exception e) {
- e.printStackTrace(LogUtil.getWarnStream(LOG));
+ });
+ if (pa)
+ threads.add(new Thread() {
+ public void run() {
+ try {
+ List res = getSeqRecords(new Path(segment,
+ CrawlDatum.PARSE_DIR_NAME), key);
+ results.put("pa", res);
+ } catch (Exception e) {
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ }
}
- }
- });
- if (pd) threads.add(new Thread() {
- public void run() {
- try {
- List res = getMapRecords(new Path(segment, ParseData.DIR_NAME), key);
- results.put("pd", res);
- } catch (Exception e) {
- e.printStackTrace(LogUtil.getWarnStream(LOG));
+ });
+ if (pd)
+ threads.add(new Thread() {
+ public void run() {
+ try {
+ List res = getMapRecords(new Path(segment,
+ ParseData.DIR_NAME), key);
+ results.put("pd", res);
+ } catch (Exception e) {
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ }
}
- }
- });
- if (pt) threads.add(new Thread() {
- public void run() {
- try {
- List res = getMapRecords(new Path(segment, ParseText.DIR_NAME), key);
- results.put("pt", res);
- } catch (Exception e) {
- e.printStackTrace(LogUtil.getWarnStream(LOG));
+ });
+ if (pt)
+ threads.add(new Thread() {
+ public void run() {
+ try {
+ List res = getMapRecords(new Path(segment,
+ ParseText.DIR_NAME), key);
+ results.put("pt", res);
+ } catch (Exception e) {
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ }
}
- }
- });
+ });
Iterator it = threads.iterator();
- while (it.hasNext()) it.next().start();
+ while (it.hasNext())
+ it.next().start();
int cnt;
do {
cnt = 0;
try {
Thread.sleep(5000);
- } catch (Exception e) {};
+ } catch (Exception e) {
+ }
+ ;
it = threads.iterator();
while (it.hasNext()) {
- if (it.next().isAlive()) cnt++;
+ if (it.next().isAlive())
+ cnt++;
}
if ((cnt > 0) && (LOG.isDebugEnabled())) {
LOG.debug("(" + cnt + " to retrieve)");
@@ -376,15 +409,16 @@
writer.flush();
}
}
-
+
private List getMapRecords(Path dir, Text key) throws Exception {
- MapFile.Reader[] readers = MapFileOutputFormat.getReaders(fs, dir, getConf());
+ MapFile.Reader[] readers = MapFileOutputFormat.getReaders(fs, dir,
+ getConf());
ArrayList res = new ArrayList();
Class keyClass = readers[0].getKeyClass();
Class valueClass = readers[0].getValueClass();
if (!keyClass.getName().equals("org.apache.hadoop.io.Text"))
throw new IOException("Incompatible key (" + keyClass.getName() + ")");
- Writable value = (Writable)valueClass.newInstance();
+ Writable value = (Writable) valueClass.newInstance();
// we don't know the partitioning schema
for (int i = 0; i < readers.length; i++) {
if (readers[i].get(key, value) != null)
@@ -395,14 +429,15 @@
}
private List getSeqRecords(Path dir, Text key) throws Exception {
- SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(getConf(), dir);
+ SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(
+ getConf(), dir);
ArrayList res = new ArrayList();
Class keyClass = readers[0].getKeyClass();
Class valueClass = readers[0].getValueClass();
if (!keyClass.getName().equals("org.apache.hadoop.io.Text"))
throw new IOException("Incompatible key (" + keyClass.getName() + ")");
- Writable aKey = (Writable)keyClass.newInstance();
- Writable value = (Writable)valueClass.newInstance();
+ Writable aKey = (Writable) keyClass.newInstance();
+ Writable value = (Writable) valueClass.newInstance();
for (int i = 0; i < readers.length; i++) {
while (readers[i].next(aKey, value)) {
if (aKey.equals(key))
@@ -422,41 +457,55 @@
public long parsed = -1L;
public long parseErrors = -1L;
}
-
+
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-
+
public void list(List dirs, Writer writer) throws Exception {
- writer.write("NAME\t\tGENERATED\tFETCHER START\t\tFETCHER END\t\tFETCHED\tPARSED\n");
+ writer
+ .write("NAME\t\tGENERATED\tFETCHER START\t\tFETCHER END\t\tFETCHED\tPARSED\n");
for (int i = 0; i < dirs.size(); i++) {
Path dir = dirs.get(i);
SegmentReaderStats stats = new SegmentReaderStats();
getStats(dir, stats);
writer.write(dir.getName() + "\t");
- if (stats.generated == -1) writer.write("?");
- else writer.write(stats.generated + "");
+ if (stats.generated == -1)
+ writer.write("?");
+ else
+ writer.write(stats.generated + "");
writer.write("\t\t");
- if (stats.start == -1) writer.write("?\t");
- else writer.write(sdf.format(new Date(stats.start)));
+ if (stats.start == -1)
+ writer.write("?\t");
+ else
+ writer.write(sdf.format(new Date(stats.start)));
writer.write("\t");
- if (stats.end == -1) writer.write("?");
- else writer.write(sdf.format(new Date(stats.end)));
+ if (stats.end == -1)
+ writer.write("?");
+ else
+ writer.write(sdf.format(new Date(stats.end)));
writer.write("\t");
- if (stats.fetched == -1) writer.write("?");
- else writer.write(stats.fetched + "");
+ if (stats.fetched == -1)
+ writer.write("?");
+ else
+ writer.write(stats.fetched + "");
writer.write("\t");
- if (stats.parsed == -1) writer.write("?");
- else writer.write(stats.parsed + "");
+ if (stats.parsed == -1)
+ writer.write("?");
+ else
+ writer.write(stats.parsed + "");
writer.write("\n");
writer.flush();
}
}
-
- public void getStats(Path segment, final SegmentReaderStats stats) throws Exception {
- SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(getConf(), new Path(segment, CrawlDatum.GENERATE_DIR_NAME));
+
+ public void getStats(Path segment, final SegmentReaderStats stats)
+ throws Exception {
+ SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(
+ getConf(), new Path(segment, CrawlDatum.GENERATE_DIR_NAME));
long cnt = 0L;
Text key = new Text();
for (int i = 0; i < readers.length; i++) {
- while (readers[i].next(key)) cnt++;
+ while (readers[i].next(key))
+ cnt++;
readers[i].close();
}
stats.generated = cnt;
@@ -466,12 +515,15 @@
long start = Long.MAX_VALUE;
long end = Long.MIN_VALUE;
CrawlDatum value = new CrawlDatum();
- MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(fs, fetchDir, getConf());
+ MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(fs, fetchDir,
+ getConf());
for (int i = 0; i < mreaders.length; i++) {
while (mreaders[i].next(key, value)) {
cnt++;
- if (value.getFetchTime() < start) start = value.getFetchTime();
- if (value.getFetchTime() > end) end = value.getFetchTime();
+ if (value.getFetchTime() < start)
+ start = value.getFetchTime();
+ if (value.getFetchTime() > end)
+ end = value.getFetchTime();
}
mreaders[i].close();
}
@@ -484,11 +536,13 @@
cnt = 0L;
long errors = 0L;
ParseData value = new ParseData();
- MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(fs, parseDir, getConf());
+ MapFile.Reader[] mreaders = MapFileOutputFormat.getReaders(fs, parseDir,
+ getConf());
for (int i = 0; i < mreaders.length; i++) {
while (mreaders[i].next(key, value)) {
cnt++;
- if (!value.getStatus().isSuccess()) errors++;
+ if (!value.getStatus().isSuccess())
+ errors++;
}
mreaders[i].close();
}
@@ -496,7 +550,7 @@
stats.parseErrors = errors;
}
}
-
+
private static final int MODE_DUMP = 0;
private static final int MODE_LIST = 1;
@@ -513,7 +567,8 @@
mode = MODE_DUMP;
else if (args[0].equals("-list"))
mode = MODE_LIST;
- else if (args[0].equals("-get")) mode = MODE_GET;
+ else if (args[0].equals("-get"))
+ mode = MODE_GET;
boolean co = true;
boolean fe = true;
@@ -545,63 +600,69 @@
}
Configuration conf = NutchConfiguration.create();
final FileSystem fs = FileSystem.get(conf);
- SegmentReader segmentReader = new SegmentReader(conf, co, fe, ge, pa, pd, pt);
+ SegmentReader segmentReader = new SegmentReader(conf, co, fe, ge, pa, pd,
+ pt);
// collect required args
switch (mode) {
- case MODE_DUMP:
- String input = args[1];
- if (input == null) {
- System.err.println("Missing required argument: ");
- usage();
- return;
- }
- String output = args.length > 2 ? args[2] : null;
- if (output == null) {
- System.err.println("Missing required argument: