Index: conf/nutch-default.xml
===================================================================
--- conf/nutch-default.xml	(revision 1562105)
+++ conf/nutch-default.xml	(working copy)
@@ -89,11 +89,18 @@
 
 <property>
   <name>http.robots.agents</name>
-  <value>*</value>
-  <description>The agent strings we'll look for in robots.txt files,
-  comma-separated, in decreasing order of precedence. You should
-  put the value of http.agent.name as the first agent name, and keep the
-  default * at the end of the list. E.g.: BlurflDev,Blurfl,*
+  <value></value>
+  <description>Any other agents, apart from 'http.agent.name', that the robots
+  parser would look for in robots.txt. Multiple agents can be provided using 
+  comma as a delimiter. eg. mybot,foo-spider,bar-crawler
+  
+  The ordering of agents does NOT matter and the robots parser would make 
+  decision based on the agent which matches first to the robots rules.  
+  Also, there is NO need to add a wildcard (ie. "*") to this string as the 
+  robots parser would smartly take care of a no-match situation. 
+    
+  If no value is specified, by default HTTP agent (ie. 'http.agent.name') 
+  would be used for user agent matching by the robots parser. 
   </description>
 </property>
 
Index: src/java/org/apache/nutch/protocol/RobotRulesParser.java
===================================================================
--- src/java/org/apache/nutch/protocol/RobotRulesParser.java	(revision 1562105)
+++ src/java/org/apache/nutch/protocol/RobotRulesParser.java	(working copy)
@@ -84,44 +84,24 @@
     this.conf = conf;
 
     // Grab the agent names we advertise to robots files.
-    String agentName = conf.get("http.agent.name");
-    if (null == agentName) {
+    this.agentNames = conf.get("http.agent.name");
+    if (this.agentNames == null || this.agentNames.trim().isEmpty()) {
       throw new RuntimeException("Agent name not configured!");
     }
 
-    String agentNames = conf.get("http.robots.agents");
-    StringTokenizer tok = new StringTokenizer(agentNames, ",");
-    ArrayList<String> agents = new ArrayList<String>();
-    while (tok.hasMoreTokens()) {
-      agents.add(tok.nextToken().trim());
-    }
-
-    /**
-     * If there are no agents for robots-parsing, use the
-     * default agent-string. If both are present, our agent-string
-     * should be the first one we advertise to robots-parsing.
-     */
-    if (agents.size() == 0) {
-      if (LOG.isErrorEnabled()) {
-        LOG.error("No agents listed in 'http.robots.agents' property!");
+    // If there are any other agents specified, append those to the list of agents
+    // Take care of remove any wildcard ie. "*" provided with 'http.robots.agents'
+    String otherAgents = conf.get("http.robots.agents");
+    if(otherAgents != null && !otherAgents.trim().isEmpty()) {
+      StringTokenizer tok = new StringTokenizer(otherAgents, ",");
+      StringBuilder sb = new StringBuilder(agentNames.trim());
+      while (tok.hasMoreTokens()) {
+        String str = tok.nextToken().trim();
+        if(str.compareTo("*") != 0)
+          sb.append(",").append(str);
       }
-    } else { 
-      StringBuffer combinedAgentsString = new StringBuffer(agentName);
-      int index = 0;
 
-      if ((agents.get(0)).equalsIgnoreCase(agentName))
-        index++;
-      else if (LOG.isErrorEnabled()) {
-        LOG.error("Agent we advertise (" + agentName
-            + ") not listed first in 'http.robots.agents' property!");
-      }
-
-      // append all the agents from the http.robots.agents property
-      for(; index < agents.size(); index++) {
-        combinedAgentsString.append(", " + agents.get(index));
-      }
-
-      this.agentNames = combinedAgentsString.toString();
+      this.agentNames = sb.toString();
     }
   }
 
@@ -137,8 +117,8 @@
    *    
    * @param url A string containing url
    * @param content Contents of the robots file in a byte array 
-   * @param contentType The 
-   * @param robotName A string containing value of  
+   * @param contentType The content type of the robots file
+   * @param robotName A string containing all the robots agent names used by parser for matching
    * @return BaseRobotRules object 
    */
   public BaseRobotRules parseRules (String url, byte[] content, String contentType, String robotName) {
@@ -160,30 +140,24 @@
   /** command-line main for testing */
   public static void main(String[] argv) {
 
-    if (argv.length < 3) {
+    if (argv.length != 3) {
       System.err.println("Usage: RobotRulesParser <robots-file> <url-file> <agent-names>\n");
       System.err.println("\tThe <robots-file> will be parsed as a robots.txt file,");
       System.err.println("\tusing the given <agent-name> to select rules.  URLs ");
       System.err.println("\twill be read (one per line) from <url-file>, and tested");
-      System.err.println("\tagainst the rules. Multiple agent names can be specified using spaces.");
+      System.err.println("\tagainst the rules. Multiple agent names can be provided using");
+      System.err.println("\tcomma as a delimiter without any spaces.");
       System.exit(-1);
     }
 
     try {
-      StringBuilder agentNames = new StringBuilder();
-      for(int counter = 2; counter < argv.length; counter++) 
-        agentNames.append(argv[counter]).append(",");
-
-      agentNames.deleteCharAt(agentNames.length()-1);
-
       byte[] robotsBytes = Files.toByteArray(new File(argv[0]));
-      BaseRobotRules rules = robotParser.parseContent(argv[0], robotsBytes, "text/plain", agentNames.toString());
+      BaseRobotRules rules = robotParser.parseContent(argv[0], robotsBytes, "text/plain", argv[2]);
 
       LineNumberReader testsIn = new LineNumberReader(new FileReader(argv[1]));
       String testPath = testsIn.readLine().trim();
       while (testPath != null) {
-        System.out.println( (rules.isAllowed(testPath) ? "allowed" : "not allowed") +
-            ":\t" + testPath);
+        System.out.println( (rules.isAllowed(testPath) ? "allowed" : "not allowed") + ":\t" + testPath);
         testPath = testsIn.readLine();
       }
       testsIn.close();