Index: src/java/org/apache/nutch/protocol/Protocol.java =================================================================== --- src/java/org/apache/nutch/protocol/Protocol.java (revision 1623865) +++ src/java/org/apache/nutch/protocol/Protocol.java (working copy) @@ -28,7 +28,7 @@ import crawlercommons.robots.BaseRobotRules; -/** A retriever of url content. Implemented by protocol extensions. */ +/** A retriever of url content. Implemented by protocol extensions. */ public interface Protocol extends Pluggable, Configurable { /** The name of the extension point. */ public final static String X_POINT_ID = Protocol.class.getName(); Index: src/java/org/apache/nutch/scoring/webgraph/WebGraph.java =================================================================== --- src/java/org/apache/nutch/scoring/webgraph/WebGraph.java (revision 1623865) +++ src/java/org/apache/nutch/scoring/webgraph/WebGraph.java (working copy) @@ -94,8 +94,8 @@ * LinkRank, is run to update scores in the Node database in a stable fashion. */ public class WebGraph - extends Configured - implements Tool { +extends Configured +implements Tool { public static final Logger LOG = LoggerFactory.getLogger(WebGraph.class); public static final String LOCK_NAME = ".locked"; @@ -110,9 +110,9 @@ * page or domain can also be limited. */ public static class OutlinkDb - extends Configured - implements Mapper, - Reducer { + extends Configured + implements Mapper, + Reducer { public static final String URL_NORMALIZING = "webgraph.url.normalizers"; public static final String URL_FILTERING = "webgraph.url.filters"; @@ -153,7 +153,7 @@ // normalize and trim the url normalized = urlNormalizers.normalize(url, - URLNormalizers.SCOPE_DEFAULT); + URLNormalizers.SCOPE_DEFAULT); normalized = normalized.trim(); } catch (Exception e) { @@ -250,8 +250,8 @@ * maps out new LinkDatum objects from new crawls ParseData. */ public void map(Text key, Writable value, - OutputCollector output, Reporter reporter) - throws IOException { + OutputCollector output, Reporter reporter) + throws IOException { // normalize url, stop processing if null String url = normalizeUrl(key.toString()); @@ -274,8 +274,8 @@ datum.getStatus() == CrawlDatum.STATUS_FETCH_REDIR_PERM || datum.getStatus() == CrawlDatum.STATUS_FETCH_GONE) { - // Tell the reducer to get rid of all instances of this key - output.collect(key, new NutchWritable(new BooleanWritable(true))); + // Tell the reducer to get rid of all instances of this key + output.collect(key, new NutchWritable(new BooleanWritable(true))); } } else if (value instanceof ParseData) { @@ -301,7 +301,7 @@ // url is existing boolean existingUrl = outlinkMap.containsKey(toUrl); if (toUrl != null - && (!existingUrl || (existingUrl && outlinkMap.get(toUrl) == null))) { + && (!existingUrl || (existingUrl && outlinkMap.get(toUrl) == null))) { outlinkMap.put(toUrl, outlink.getAnchor()); } } @@ -328,8 +328,8 @@ } public void reduce(Text key, Iterator values, - OutputCollector output, Reporter reporter) - throws IOException { + OutputCollector output, Reporter reporter) + throws IOException { // aggregate all outlinks, get the most recent timestamp for a fetch // which should be the timestamp for all of the most recent outlinks @@ -381,10 +381,10 @@ // outlinks must be the most recent and conform to internal url and // limiting rules, if it does collect it if (datum.getTimestamp() == mostRecent - && (!limitPages || (limitPages && !pages.contains(toPage))) - && (!limitDomains || (limitDomains && !domains.contains(toDomain))) - && (!ignoreHost || (ignoreHost && !toHost.equalsIgnoreCase(host))) - && (!ignoreDomain || (ignoreDomain && !toDomain.equalsIgnoreCase(domain)))) { + && (!limitPages || (limitPages && !pages.contains(toPage))) + && (!limitDomains || (limitDomains && !domains.contains(toDomain))) + && (!ignoreHost || (ignoreHost && !toHost.equalsIgnoreCase(host))) + && (!ignoreDomain || (ignoreDomain && !toDomain.equalsIgnoreCase(domain)))) { output.collect(key, datum); pages.add(toPage); domains.add(toDomain); @@ -402,8 +402,8 @@ * updated. */ private static class InlinkDb - extends Configured - implements Mapper { + extends Configured + implements Mapper { private long timestamp; @@ -423,8 +423,8 @@ * new system timestamp, type and to and from url switched. */ public void map(Text key, LinkDatum datum, - OutputCollector output, Reporter reporter) - throws IOException { + OutputCollector output, Reporter reporter) + throws IOException { // get the to and from url and the anchor String fromUrl = key.toString(); @@ -443,8 +443,8 @@ * for each url and a score slot for analysis programs such as LinkRank. */ private static class NodeDb - extends Configured - implements Reducer { + extends Configured + implements Reducer { /** * Configures job. @@ -458,8 +458,8 @@ * score of 0.0 for each url (node) in the webgraph. */ public void reduce(Text key, Iterator values, - OutputCollector output, Reporter reporter) - throws IOException { + OutputCollector output, Reporter reporter) + throws IOException { Node node = new Node(); int numInlinks = 0; @@ -498,7 +498,7 @@ * @throws IOException If an error occurs while processing the WebGraph. */ public void createWebGraph(Path webGraphDb, Path[] segments, boolean normalize, boolean filter) - throws IOException { + throws IOException { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); long start = System.currentTimeMillis(); @@ -529,7 +529,7 @@ } Path tempOutlinkDb = new Path(outlinkDb + "-" - + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); JobConf outlinkJob = new NutchJob(conf); outlinkJob.setJobName("Outlinkdb: " + outlinkDb); @@ -588,7 +588,7 @@ LOG.info("OutlinkDb: finished"); } catch (IOException e) { - + // remove lock file and and temporary directory if an error occurs LockUtil.removeLockFile(fs, lock); if (fs.exists(tempOutlinkDb)) { @@ -601,7 +601,7 @@ // inlink and temp link database paths Path inlinkDb = new Path(webGraphDb, INLINK_DIR); Path tempInlinkDb = new Path(inlinkDb + "-" - + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); JobConf inlinkJob = new NutchJob(conf); inlinkJob.setJobName("Inlinkdb " + inlinkDb); @@ -618,7 +618,7 @@ inlinkJob.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", false); try { - + // run the inlink and replace any old with new LOG.info("InlinkDb: running"); JobClient.runJob(inlinkJob); @@ -627,7 +627,7 @@ LOG.info("InlinkDb: finished"); } catch (IOException e) { - + // remove lock file and and temporary directory if an error occurs LockUtil.removeLockFile(fs, lock); if (fs.exists(tempInlinkDb)) { @@ -640,7 +640,7 @@ // node and temp node database paths Path nodeDb = new Path(webGraphDb, NODE_DIR); Path tempNodeDb = new Path(nodeDb + "-" - + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); JobConf nodeJob = new NutchJob(conf); nodeJob.setJobName("NodeDb " + nodeDb); @@ -659,7 +659,7 @@ nodeJob.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", false); try { - + // run the node job and replace old nodedb with new LOG.info("NodeDb: running"); JobClient.runJob(nodeJob); @@ -668,7 +668,7 @@ LOG.info("NodeDb: finished"); } catch (IOException e) { - + // remove lock file and and temporary directory if an error occurs LockUtil.removeLockFile(fs, lock); if (fs.exists(tempNodeDb)) { @@ -686,7 +686,7 @@ } public static void main(String[] args) - throws Exception { + throws Exception { int res = ToolRunner.run(NutchConfiguration.create(), new WebGraph(), args); System.exit(res); } @@ -695,50 +695,43 @@ * Parses command link arguments and runs the WebGraph jobs. */ public int run(String[] args) - throws Exception { + throws Exception { + //boolean options + Option helpOpt = new Option("h", "help", false, "show this help message"); + Option normOpt = new Option("n", "normalize", false, "whether to use URLNormalizers on the URL's in the segment"); + Option filtOpt = new Option("f", "filter", false, "whether to use URLFilters on the URL's in the segment"); + + //argument options + @SuppressWarnings("static-access") + Option graphOpt = OptionBuilder.withArgName("webgraphdb") + .hasArg().withDescription("the web graph database to create (if none exists) or use if one does") + .create("webgraphdb"); + @SuppressWarnings("static-access") + Option segOpt = OptionBuilder.withArgName("segment") + .hasArgs().withDescription("the segment(s) to use") + .create("segment"); + @SuppressWarnings("static-access") + Option segDirOpt = OptionBuilder.withArgName("segmentDir") + .hasArgs().withDescription("the segment directory to use") + .create("segmentDir"); + + //create the options Options options = new Options(); - OptionBuilder.withArgName("help"); - OptionBuilder.withDescription("show this help message"); - Option helpOpts = OptionBuilder.create("help"); - options.addOption(helpOpts); - - OptionBuilder.withArgName("webgraphdb"); - OptionBuilder.hasArg(); - OptionBuilder.withDescription("the web graph database to use"); - Option webGraphDbOpts = OptionBuilder.create("webgraphdb"); - options.addOption(webGraphDbOpts); - - OptionBuilder.withArgName("segment"); - OptionBuilder.hasArgs(); - OptionBuilder.withDescription("the segment(s) to use"); - Option segOpts = OptionBuilder.create("segment"); - options.addOption(segOpts); - - OptionBuilder.withArgName("segmentDir"); - OptionBuilder.hasArgs(); - OptionBuilder.withDescription("the segment directory to use"); - Option segDirOpts = OptionBuilder.create("segmentDir"); - options.addOption(segDirOpts); - - OptionBuilder.withArgName("normalize"); - OptionBuilder.withDescription("whether to use URLNormalizers on the URL's in the segment"); - Option normalizeOpts = OptionBuilder.create("normalize"); - options.addOption(normalizeOpts); - - OptionBuilder.withArgName("filter"); - OptionBuilder.withDescription("whether to use URLFilters on the URL's in the segment"); - Option filterOpts = OptionBuilder.create("filter"); - options.addOption(filterOpts); + options.addOption(helpOpt); + options.addOption(normOpt); + options.addOption(filtOpt); + options.addOption(graphOpt); + options.addOption(segOpt); + options.addOption(segDirOpt); CommandLineParser parser = new GnuParser(); try { - CommandLine line = parser.parse(options, args); if (line.hasOption("help") || !line.hasOption("webgraphdb") - || (!line.hasOption("segment") && !line.hasOption("segmentDir"))) { + || (!line.hasOption("segment") && !line.hasOption("segmentDir"))) { HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("WebGraph", options); + formatter.printHelp("WebGraph", options, true); return -1; }