Index: Fetcher.java
===================================================================
--- Fetcher.java	(revision 1290966)
+++ Fetcher.java	(working copy)
@@ -16,7 +16,9 @@
  */
 package org.apache.nutch.fetcher;
 
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -113,6 +115,107 @@
     }
   }
 
+  String hostsTimeout_normHost(String host){
+	  	try{
+	  		if("www.".compareTo(host.substring(0, 4))==0)
+	  			return host.substring(4);
+	  	}
+	  	catch(Exception e){}
+	  	return host;
+	  }
+	  public int hostsTimeout_getTimeout(String url){
+	  	//If host is not in hostsTimeout, add <host , minTimeout> to hostsTimeout
+		  try{
+			  String host=hostsTimeout_normHost(new URL(url).getHost());
+			  Double timeout=hostsTimeout_hostsTimeout.get(host);
+			  if(timeout==null){
+				  hostsTimeout_hostsTimeout.put(host, hostsTimeout_minTimeout);
+				  return (int)hostsTimeout_minTimeout;
+			  }
+			  return timeout.intValue();
+		  }
+		  catch(Exception ee){
+			  return (int)hostsTimeout_minTimeout;
+		  }
+	  }
+	  public void hostsTimeout_incrementTimeout(String url){
+		  try{
+			  String host=hostsTimeout_normHost(new URL(url).getHost());
+			  double timeout=hostsTimeout_hostsTimeout.get(host).doubleValue();
+			  timeout=timeout+hostsTimeout_inc;
+			  if (timeout > hostsTimeout_maxTimeout)
+				  timeout= hostsTimeout_maxTimeout;
+			  hostsTimeout_hostsTimeout.put(host, new Double(timeout));
+		  }
+		  catch(Exception ee){
+		  }
+	  }
+	  public void hostsTimeout_decrementTimeout(String url){
+		  try{
+			  String host=hostsTimeout_normHost(new URL(url).getHost());
+			  double timeout=hostsTimeout_hostsTimeout.get(host).doubleValue();
+			  timeout=timeout-hostsTimeout_dec;
+			  if (timeout < hostsTimeout_minTimeout)
+				  timeout= hostsTimeout_minTimeout;
+			  hostsTimeout_hostsTimeout.put(host, new Double(timeout));
+		  }
+		  catch(Exception ee){
+		  }
+	  }
+	  void hostsTimeout_loadFromFile(){
+		    String fileName="/nutch/nutch-1.3/runtime/local/logs/hosts_timeout_file";
+			java.io.File f=new java.io.File(fileName);
+	  	try{
+				if(!f.exists())
+					f.createNewFile();
+	  	}catch(Exception ipo){return;}
+			try{
+				java.io.FileInputStream fin=new java.io.FileInputStream(f);
+				java.io.DataInputStream in = new java.io.DataInputStream(fin);
+				java.io.BufferedReader br = new java.io.BufferedReader(new InputStreamReader(in));
+				String strLine;
+				while ((strLine = br.readLine()) != null)   {
+					int i1=strLine.indexOf('*');
+					String host=strLine.substring(0,i1);
+					double timeout=hostsTimeout_minTimeout;
+					try{
+						timeout=Double.valueOf(strLine.substring(i1+1)).doubleValue();
+					}catch(Exception e){}
+					hostsTimeout_hostsTimeout.put(host, new Double(timeout));
+				}
+				fin.close();
+	  	}
+			catch(Exception iuu){}
+	  }
+	  void hostsTimeout_saveFile(){
+		  String fileName="/nutch/nutch-1.3/runtime/local/logs/hosts_timeout_file";
+		  Set set = hostsTimeout_hostsTimeout.entrySet();
+		  Iterator it = set.iterator();
+	  	
+		  java.io.File f=new java.io.File(fileName);
+		  try{
+				if(!f.exists())
+					f.createNewFile();
+		  }catch(Exception ipo){}
+		  try{
+				java.io.FileOutputStream fs=new java.io.FileOutputStream(f,false);
+				fs.write("".getBytes());
+				fs=new FileOutputStream(f,true);
+		    	while(it.hasNext()) {
+		        	Map.Entry entry = (Map.Entry)it.next();
+		        	String host=entry.getKey()+"*"+entry.getValue()+"\n";
+					fs.write(host.getBytes());	
+		        } 
+		    	fs.close();
+	  	}
+			catch(Exception iuu){}
+	  }
+	  private java.util.concurrent.ConcurrentHashMap<String,Double> hostsTimeout_hostsTimeout = new java.util.concurrent.ConcurrentHashMap<String,Double>();
+	  private double hostsTimeout_minTimeout=10000;
+	  private double hostsTimeout_maxTimeout=4*hostsTimeout_minTimeout;
+	  private double hostsTimeout_inc=1000;
+	  private double hostsTimeout_dec=500;
+	  
   private OutputCollector<Text, NutchWritable> output;
   private Reporter reporter;
 
@@ -687,7 +790,7 @@
                   fiq.crawlDelay = rules.getCrawlDelay();
                 }
               }
-              ProtocolOutput output = protocol.getProtocolOutput(fit.url, fit.datum);
+              ProtocolOutput output = protocol.getProtocolOutput(fit.url, fit.datum,hostsTimeout_getTimeout(fit.url.toString()));
               ProtocolStatus status = output.getStatus();
               Content content = output.getContent();
               ParseStatus pstatus = null;
@@ -706,6 +809,7 @@
                 break;
 
               case ProtocolStatus.SUCCESS:        // got a page
+            	  hostsTimeout_decrementTimeout(fit.url.toString());
                 pstatus = output(fit.url, fit.datum, content, status, CrawlDatum.STATUS_FETCH_SUCCESS, fit.outlinkDepth);
                 updateStatus(content.getContent().length);
                 if (pstatus != null && pstatus.isSuccess() &&
@@ -785,6 +889,7 @@
                 break;
 
               case ProtocolStatus.EXCEPTION:
+            	  hostsTimeout_incrementTimeout(fit.url.toString());
                 logError(fit.url, status.getMessage());
                 int killedURLs = fetchQueues.checkExceptionThreshold(fit.getQueueID());
                 if (killedURLs!=0)
@@ -792,6 +897,7 @@
                 /* FALLTHROUGH */
               case ProtocolStatus.RETRY:          // retry
               case ProtocolStatus.BLOCKED:
+            	  hostsTimeout_incrementTimeout(fit.url.toString());
                 output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_FETCH_RETRY);
                 break;
 
@@ -1087,9 +1193,15 @@
 
   }
 
-  public Fetcher() { super(null); }
+  public Fetcher() { 
+	  super(null);
+	  hostsTimeout_loadFromFile();
+  }
 
-  public Fetcher(Configuration conf) { super(conf); }
+  public Fetcher(Configuration conf) { 
+	  super(conf);
+	  hostsTimeout_loadFromFile();
+  }
 
   private void updateStatus(int bytesInPage) throws IOException {
     pages.incrementAndGet();
@@ -1242,12 +1354,13 @@
         if (LOG.isWarnEnabled()) {
           LOG.warn("Aborting with "+activeThreads+" hung threads.");
         }
+        hostsTimeout_saveFile();
         return;
       }
 
     } while (activeThreads.get() > 0);
     LOG.info("-activeThreads=" + activeThreads);
-
+    hostsTimeout_saveFile();
   }
 
   public void fetch(Path segment, int threads)