Index: src/test/org/apache/nutch/fetcher/TestFetcher.java =================================================================== --- src/test/org/apache/nutch/fetcher/TestFetcher.java (revision 508235) +++ src/test/org/apache/nutch/fetcher/TestFetcher.java (working copy) @@ -69,6 +69,42 @@ fs.delete(testdir); } + public void testAgentNameCheck() { + + boolean failedNoAgentName = false; + conf.set("http.agent.name", ""); + + try { + + //generate seedlist + ArrayList urls=new ArrayList(); + addUrl(urls,"index.html"); + CrawlDBTestUtil.generateSeedList(fs, urlPath, urls); + + //inject + Injector injector=new Injector(conf); + injector.inject(crawldbPath, urlPath); + + //generate + Generator g=new Generator(conf); + Path generatedSegment = g.generate(crawldbPath, segmentsPath, 1, + Long.MAX_VALUE, Long.MAX_VALUE, false, false); + + conf.setBoolean("fetcher.parse", true); + Fetcher fetcher=new Fetcher(conf); + fetcher.fetch(generatedSegment, 1); + } + catch (IllegalArgumentException iae) { + String message = iae.getMessage(); + failedNoAgentName = message.equals("Fetcher: No agents listed in " + + "'http.agent.name' property."); + } + catch (Exception e) { + } + + assertTrue(failedNoAgentName); + } + public void testFetch() throws IOException { Index: src/test/crawl-tests.xml =================================================================== --- src/test/crawl-tests.xml (revision 509080) +++ src/test/crawl-tests.xml (working copy) @@ -28,4 +28,19 @@ test-nutch + + http.agent.name.check + true + + + + http.robots.agents + test-nutch,* + The agent strings we'll look for in robots.txt files, + comma-separated, in decreasing order of precedence. You should + put the value of http.agent.name as the first agent name, and keep the + default * at the end of the list. E.g.: BlurflDev,Blurfl,* + + + \ No newline at end of file Index: src/java/org/apache/nutch/fetcher/Fetcher.java =================================================================== --- src/java/org/apache/nutch/fetcher/Fetcher.java (revision 509081) +++ src/java/org/apache/nutch/fetcher/Fetcher.java (working copy) @@ -18,6 +18,8 @@ package org.apache.nutch.fetcher; import java.io.IOException; +import java.util.ArrayList; +import java.util.StringTokenizer; // Commons Logging imports import org.apache.commons.logging.Log; @@ -370,6 +372,42 @@ reporter.setStatus(status); } + private void checkConfiguration() { + + // ensure that a value has been set for the agent name and that that + // agent name is the first value in the agents we advertise for robot + // rules parsing + String agentName = getConf().get("http.agent.name"); + if (agentName == null || agentName.trim().length() == 0) { + String message = "Fetcher: No agents listed in 'http.agent.name'" + + " property."; + if (LOG.isFatalEnabled()) { + LOG.fatal(message); + } + throw new IllegalArgumentException(message); + } + else { + + // get all of the agents that we advertise + String agentNames = getConf().get("http.robots.agents"); + StringTokenizer tok = new StringTokenizer(agentNames, ","); + ArrayList agents = new ArrayList(); + while (tok.hasMoreTokens()) { + agents.add(tok.nextToken().trim()); + } + + // if the first one is not equal to our agent name, log fatal and throw + // an exception + if (!(agents.get(0)).equalsIgnoreCase(agentName)) { + String message = "Fetcher: Your 'http.agent.name' value should be " + + "listed first in 'http.robots.agents' property."; + if (LOG.isWarnEnabled()) { + LOG.warn(message); + } + } + } + } + public Fetcher() { } @@ -443,6 +481,8 @@ public void fetch(Path segment, int threads) throws IOException { + checkConfiguration(); + if (LOG.isInfoEnabled()) { LOG.info("Fetcher: starting"); LOG.info("Fetcher: segment: " + segment); Index: src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java =================================================================== --- src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (revision 508235) +++ src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (working copy) @@ -223,33 +223,12 @@ // Grab the agent names we advertise to robots files. // String agentName = conf.get("http.agent.name"); - if (null == agentName) { - throw new RuntimeException("Agent name not configured!"); - } String agentNames = conf.get("http.robots.agents"); StringTokenizer tok = new StringTokenizer(agentNames, ","); ArrayList agents = new ArrayList(); while (tok.hasMoreTokens()) { agents.add(tok.nextToken().trim()); } - - // - // If there are no agents for robots-parsing, use our - // default agent-string. If both are present, our agent-string - // should be the first one we advertise to robots-parsing. - // - if (agents.size() == 0) { - agents.add(agentName); - if (LOG.isFatalEnabled()) { - LOG.fatal("No agents listed in 'http.robots.agents' property!"); - } - } else if (!((String)agents.get(0)).equalsIgnoreCase(agentName)) { - agents.add(0, agentName); - if (LOG.isFatalEnabled()) { - LOG.fatal("Agent we advertise (" + agentName - + ") not listed first in 'http.robots.agents' property!"); - } - } setRobotNames((String[]) agents.toArray(new String[agents.size()])); }