Index: src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
===================================================================
--- src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (revision 367595)
+++ src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (working copy)
@@ -43,35 +43,46 @@
public static final int BUFFER_SIZE = 8 * 1024;
- public static String PROXY_HOST =
- NutchConf.get().get("http.proxy.host");
-
- public static int PROXY_PORT =
- NutchConf.get().getInt("http.proxy.port", 8080);
-
- public static boolean PROXY =
- (PROXY_HOST != null && PROXY_HOST.length() > 0);
-
- public static int TIMEOUT =
- NutchConf.get().getInt("http.timeout", 10000);
-
- public static int MAX_CONTENT =
- NutchConf.get().getInt("http.content.limit", 64 * 1024);
-
- public static int MAX_DELAYS =
- NutchConf.get().getInt("http.max.delays", 3);
-
- public static int MAX_THREADS_PER_HOST =
- NutchConf.get().getInt("fetcher.threads.per.host", 1);
-
- public static String AGENT_STRING =
- getAgentString();
-
- public static long SERVER_DELAY =
- (long) (NutchConf.get().getFloat("fetcher.server.delay", 1.0f) * 1000);
-
+ private static final byte[] EMPTY_CONTENT = new byte[0];
- private static final byte[] EMPTY_CONTENT = new byte[0];
+ private RobotRulesParser robots = null;
+
+ /** The proxy hostname. */
+ protected String proxyHost = null;
+
+ /** The proxy port. */
+ protected int proxyPort = 8080;
+
+ /** Indicates if a proxy is used */
+ protected boolean useProxy = false;
+
+ /** The network timeout in millisecond */
+ protected int timeout = 10000;
+
+ /** The length limit for downloaded content, in bytes. */
+ protected int maxContent = 64 * 1024;
+
+ /** The number of times a thread will delay when trying to fetch a page. */
+ protected int maxDelays = 3;
+
+ /**
+ * The maximum number of threads that should be allowed
+ * to access a host at one time.
+ */
+ protected int maxThreadsPerHost = 1;
+
+ /**
+ * The number of seconds the fetcher will delay between
+ * successive requests to the same server.
+ */
+ protected long serverDelay = 1000;
+
+ /** The Nutch 'User-Agent' request header */
+ protected String userAgent = getAgentString(
+ "NutchCVS", null, "Nutch",
+ "http://lucene.apache.org/nutch/bot.html",
+ "nutch-agent@lucene.apache.org");
+
/**
* Maps from InetAddress to a Long naming the time it should be unblocked.
@@ -97,7 +108,10 @@
/** The specified logger */
private Logger logger = LOGGER;
-
+
+ /** The nutch configuration */
+ private NutchConf conf = null;
+
/** Creates a new instance of HttpBase */
public HttpBase() {
@@ -109,15 +123,42 @@
if (logger != null) {
this.logger = logger;
}
- logger.info("http.proxy.host = " + PROXY_HOST);
- logger.info("http.proxy.port = " + PROXY_PORT);
- logger.info("http.timeout = " + TIMEOUT);
- logger.info("http.content.limit = " + MAX_CONTENT);
- logger.info("http.agent = " + AGENT_STRING);
- logger.info("fetcher.server.delay = " + SERVER_DELAY);
- logger.info("http.max.delays = " + MAX_DELAYS);
+ robots = new RobotRulesParser();
+ logConf();
}
-
+
+
+ /* ------------------------- *
+ * *
+ * ------------------------- */
+
+ // Inherited Javadoc
+ public void setConf(NutchConf conf) {
+ this.conf = conf;
+ this.proxyHost = conf.get("http.proxy.host");
+ this.proxyPort = conf.getInt("http.proxy.port", 8080);
+ this.useProxy = (proxyHost != null && proxyHost.length() > 0);
+ this.timeout = conf.getInt("http.timeout", 10000);
+ this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
+ this.maxDelays = conf.getInt("http.max.delays", 3);
+ this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
+ this.userAgent = getAgentString(conf.get("http.agent.name"),
+ conf.get("http.agent.version"),
+ conf.get("http.agent.description"),
+ conf.get("http.agent.url"),
+ conf.get("http.agent.email"));
+ this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
+ this.robots.setConf(conf);
+ logConf();
+ }
+
+ // Inherited Javadoc
+ public NutchConf getConf() {
+ return this.conf;
+ }
+
+
+ // Inherited Javadoc
public ProtocolOutput getProtocolOutput(UTF8 url, CrawlDatum datum) {
String urlString = url.toString();
@@ -125,7 +166,7 @@
URL u = new URL(urlString);
try {
- if (!RobotRulesParser.isAllowed(this, u)) {
+ if (!robots.isAllowed(this, u)) {
return new ProtocolOutput(null, new ProtocolStatus(ProtocolStatus.ROBOTS_DENIED, url));
}
} catch (Throwable e) {
@@ -146,7 +187,7 @@
Content c = new Content(u.toString(), u.toString(),
(content == null ? EMPTY_CONTENT : content),
response.getHeader("Content-Type"),
- response.getHeaders());
+ response.getHeaders(), this.conf);
if (code == 200) { // got a good response
return new ProtocolOutput(c); // return it
@@ -203,8 +244,49 @@
}
}
-
- private static InetAddress blockAddr(URL url) throws ProtocolException {
+ /* -------------------------- *
+ * *
+ * -------------------------- */
+
+
+ public String getProxyHost() {
+ return proxyHost;
+ }
+
+ public int getProxyPort() {
+ return proxyPort;
+ }
+
+ public boolean useProxy() {
+ return useProxy;
+ }
+
+ public int getTimeout() {
+ return timeout;
+ }
+
+ public int getMaxContent() {
+ return maxContent;
+ }
+
+ public int getMaxDelays() {
+ return maxDelays;
+ }
+
+ public int getMaxThreadsPerHost() {
+ return maxThreadsPerHost;
+ }
+
+ public long getServerDelay() {
+ return serverDelay;
+ }
+
+ public String getUserAgent() {
+ return userAgent;
+ }
+
+
+ private InetAddress blockAddr(URL url) throws ProtocolException {
InetAddress addr;
try {
@@ -229,21 +311,21 @@
count++; // increment & store
THREADS_PER_HOST_COUNT.put(addr, new Integer(count));
- if (count >= MAX_THREADS_PER_HOST) {
+ if (count >= maxThreadsPerHost) {
BLOCKED_ADDR_TO_TIME.put(addr, new Long(0)); // block it
}
return addr;
}
}
- if (delays == MAX_DELAYS)
+ if (delays == maxDelays)
throw new HttpException("Exceeded http.max.delays: retry later.");
long done = time.longValue();
long now = System.currentTimeMillis();
long sleep = 0;
if (done == 0) { // address is still in use
- sleep = SERVER_DELAY; // wait at least delay
+ sleep = serverDelay; // wait at least delay
} else if (now < done) { // address is on hold
sleep = done - now; // wait until its free
@@ -256,14 +338,14 @@
}
}
- private static void unblockAddr(InetAddress addr) {
+ private void unblockAddr(InetAddress addr) {
synchronized (BLOCKED_ADDR_TO_TIME) {
int addrCount = ((Integer)THREADS_PER_HOST_COUNT.get(addr)).intValue();
if (addrCount == 1) {
THREADS_PER_HOST_COUNT.remove(addr);
BLOCKED_ADDR_QUEUE.addFirst(addr);
BLOCKED_ADDR_TO_TIME.put
- (addr, new Long(System.currentTimeMillis()+SERVER_DELAY));
+ (addr, new Long(System.currentTimeMillis() + serverDelay));
} else {
THREADS_PER_HOST_COUNT.put(addr, new Integer(addrCount - 1));
}
@@ -285,14 +367,12 @@
}
}
- private static String getAgentString() {
+ private static String getAgentString(String agentName,
+ String agentVersion,
+ String agentDesc,
+ String agentURL,
+ String agentEmail) {
- String agentName = NutchConf.get().get("http.agent.name");
- String agentVersion = NutchConf.get().get("http.agent.version");
- String agentDesc = NutchConf.get().get("http.agent.description");
- String agentURL = NutchConf.get().get("http.agent.url");
- String agentEmail = NutchConf.get().get("http.agent.email");
-
if ( (agentName == null) || (agentName.trim().length() == 0) )
LOGGER.severe("No User-Agent string set (http.agent.name)!");
@@ -327,6 +407,16 @@
}
return buf.toString();
}
+
+ protected void logConf() {
+ logger.info("http.proxy.host = " + proxyHost);
+ logger.info("http.proxy.port = " + proxyPort);
+ logger.info("http.timeout = " + timeout);
+ logger.info("http.content.limit = " + maxContent);
+ logger.info("http.agent = " + userAgent);
+ logger.info("fetcher.server.delay = " + serverDelay);
+ logger.info("http.max.delays = " + maxDelays);
+ }
protected static void main(HttpBase http, String[] args) throws Exception {
boolean verbose = false;
@@ -341,7 +431,7 @@
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].equals("-timeout")) { // found -timeout option
- TIMEOUT = Integer.parseInt(args[++i]) * 1000;
+ http.timeout = Integer.parseInt(args[++i]) * 1000;
} else if (args[i].equals("-verbose")) { // found -verbose option
verbose = true;
} else if (i != args.length - 1) {
Index: src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java
===================================================================
--- src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (revision 367595)
+++ src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (working copy)
@@ -30,6 +30,7 @@
// Nutch imports
import org.apache.nutch.util.NutchConf;
+import org.apache.nutch.util.NutchConfigurable;
import org.apache.nutch.util.LogFormatter;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
@@ -45,14 +46,13 @@
* @author Mike Cafarella
* @author Doug Cutting
*/
-public class RobotRulesParser {
+public class RobotRulesParser implements NutchConfigurable {
+
public static final Logger LOG=
- LogFormatter.getLogger("org.apache.nutch.fetcher.RobotRulesParser");
+ LogFormatter.getLogger(RobotRulesParser.class.getName());
- private static final boolean ALLOW_FORBIDDEN =
- NutchConf.get().getBoolean("http.robots.403.allow", false);
+ private boolean allowForbidden = false;
- private static final String[] AGENTS = getAgents();
private static final Hashtable CACHE = new Hashtable();
private static final String CHARACTER_ENCODING= "UTF-8";
@@ -60,9 +60,9 @@
private static final RobotRuleSet EMPTY_RULES= new RobotRuleSet();
- private static RobotRuleSet FORBID_ALL_RULES =
- new RobotRulesParser().getForbidAllRules();
+ private static RobotRuleSet FORBID_ALL_RULES = getForbidAllRules();
+ private NutchConf conf;
private HashMap robotNames;
/**
@@ -87,15 +87,7 @@
}
/**
- * should not be instantiated from outside RobotRulesParser
*/
- private RobotRuleSet() {
- tmpEntries= new ArrayList();
- entries= null;
- }
-
- /**
- */
private void addPrefix(String prefix, boolean allow) {
if (tmpEntries == null) {
tmpEntries= new ArrayList();
@@ -182,14 +174,25 @@
}
- public RobotRulesParser() { this(AGENTS); }
+ RobotRulesParser() { }
- private static String[] getAgents() {
+ public RobotRulesParser(NutchConf conf) {
+ setConf(conf);
+ }
+
+
+ /* ---------------------------------- *
+ * *
+ * ---------------------------------- */
+
+ public void setConf(NutchConf conf) {
+ this.conf = conf;
+ allowForbidden = conf.getBoolean("http.robots.403.allow", false);
//
// Grab the agent names we advertise to robots files.
//
- String agentName = NutchConf.get().get("http.agent.name");
- String agentNames = NutchConf.get().get("http.robots.agents");
+ String agentName = conf.get("http.agent.name");
+ String agentNames = conf.get("http.robots.agents");
StringTokenizer tok = new StringTokenizer(agentNames, ",");
ArrayList agents = new ArrayList();
while (tok.hasMoreTokens()) {
@@ -197,23 +200,39 @@
}
//
- // If there are no agents for robots-parsing, use our
+ // If there are no agents for robots-parsing, use our
// default agent-string. If both are present, our agent-string
// should be the first one we advertise to robots-parsing.
- //
+ //
if (agents.size() == 0) {
agents.add(agentName);
LOG.severe("No agents listed in 'http.robots.agents' property!");
} else if (!((String)agents.get(0)).equalsIgnoreCase(agentName)) {
agents.add(0, agentName);
- LOG.severe("Agent we advertise (" + agentName
+ LOG.severe("Agent we advertise (" + agentName
+ ") not listed first in 'http.robots.agents' property!");
}
+ setRobotNames((String[]) agents.toArray(new String[agents.size()]));
+ }
- return (String[])agents.toArray(new String[agents.size()]);
+ public NutchConf getConf() {
+ return conf;
}
+ /* ---------------------------------- *
+ * *
+ * ---------------------------------- */
+ private void setRobotNames(String[] robotNames) {
+ this.robotNames= new HashMap();
+ for (int i= 0; i < robotNames.length; i++) {
+ this.robotNames.put(robotNames[i].toLowerCase(), new Integer(i));
+ }
+ // always make sure "*" is included
+ if (!this.robotNames.containsKey("*"))
+ this.robotNames.put("*", new Integer(robotNames.length));
+ }
+
/**
* Creates a new RobotRulesParser which will use the
* supplied robotNames when choosing which stanza to
@@ -223,14 +242,8 @@
* rules associated with the robot name having the smallest index
* will be used.
*/
- public RobotRulesParser(String[] robotNames) {
- this.robotNames= new HashMap();
- for (int i= 0; i < robotNames.length; i++) {
- this.robotNames.put(robotNames[i].toLowerCase(), new Integer(i));
- }
- // always make sure "*" is included
- if (!this.robotNames.containsKey("*"))
- this.robotNames.put("*", new Integer(robotNames.length));
+ RobotRulesParser(String[] robotNames) {
+ setRobotNames(robotNames);
}
/**
@@ -368,7 +381,7 @@
return rules;
}
- public static boolean isAllowed(HttpBase http, URL url)
+ public boolean isAllowed(HttpBase http, URL url)
throws ProtocolException, IOException {
String host = url.getHost();
@@ -382,8 +395,8 @@
new CrawlDatum(), true);
if (response.getCode() == 200) // found rules: parse them
- robotRules = new RobotRulesParser().parseRules(response.getContent());
- else if ( (response.getCode() == 403) && (!ALLOW_FORBIDDEN) )
+ robotRules = parseRules(response.getContent());
+ else if ( (response.getCode() == 403) && (!allowForbidden) )
robotRules = FORBID_ALL_RULES; // use forbid all
else
robotRules = EMPTY_RULES; // use default rules
Index: src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
===================================================================
--- src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java (revision 367595)
+++ src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java (working copy)
@@ -33,25 +33,29 @@
public class Http extends HttpBase {
public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.net.Http");
+ LogFormatter.getLogger(Http.class.getName());
- static {
- if (NutchConf.get().getBoolean("http.verbose", false))
- LOG.setLevel(Level.FINE);
- }
-
public Http() {
super(LOG);
}
+ public void setConf(NutchConf conf) {
+ super.setConf(conf);
+ Level logLevel = Level.WARNING;
+ if (conf.getBoolean("http.verbose", false)) {
+ logLevel = Level.FINE;
+ }
+ LOG.setLevel(logLevel);
+ }
+
public static void main(String[] args) throws Exception {
main(new Http(), args);
}
protected Response getResponse(URL url, CrawlDatum datum, boolean redirect)
throws ProtocolException, IOException {
- return new HttpResponse(url, datum);
+ return new HttpResponse(this, url, datum);
}
}
Index: src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
===================================================================
--- src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java (revision 367595)
+++ src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java (working copy)
@@ -35,13 +35,15 @@
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ContentProperties;
import org.apache.nutch.protocol.ProtocolException;
+import org.apache.nutch.protocol.http.api.HttpBase;
import org.apache.nutch.protocol.http.api.HttpException;
import org.apache.nutch.util.GZIPUtils;
/** An HTTP response. */
public class HttpResponse implements Response {
-
+
+ private HttpBase http;
private URL url;
private String orig;
private String base;
@@ -50,9 +52,10 @@
private ContentProperties headers = new ContentProperties();
- public HttpResponse(URL url, CrawlDatum datum)
+ public HttpResponse(HttpBase http, URL url, CrawlDatum datum)
throws ProtocolException, IOException {
+ this.http = http;
this.url = url;
this.orig = url.toString();
this.base = url.toString();
@@ -83,20 +86,20 @@
try {
socket = new Socket(); // create the socket
- socket.setSoTimeout(Http.TIMEOUT);
+ socket.setSoTimeout(http.getTimeout());
// connect
- String sockHost = Http.PROXY ? Http.PROXY_HOST : host;
- int sockPort = Http.PROXY ? Http.PROXY_PORT : port;
+ String sockHost = http.useProxy() ? http.getProxyHost() : host;
+ int sockPort = http.useProxy() ? http.getProxyPort() : port;
InetSocketAddress sockAddr= new InetSocketAddress(sockHost, sockPort);
- socket.connect(sockAddr, Http.TIMEOUT);
+ socket.connect(sockAddr, http.getTimeout());
// make request
OutputStream req = socket.getOutputStream();
StringBuffer reqStr = new StringBuffer("GET ");
- if(Http.PROXY){
+ if (http.useProxy()) {
reqStr.append(url.getProtocol()+"://"+host+portString+path);
} else {
reqStr.append(path);
@@ -111,11 +114,12 @@
reqStr.append("Accept-Encoding: x-gzip, gzip\r\n");
- if ((Http.AGENT_STRING == null) || (Http.AGENT_STRING.length() == 0)) {
+ String userAgent = http.getUserAgent();
+ if ((userAgent == null) || (userAgent.length() == 0)) {
Http.LOG.severe("User-agent is not set!");
} else {
reqStr.append("User-Agent: ");
- reqStr.append(Http.AGENT_STRING);
+ reqStr.append(userAgent);
reqStr.append("\r\n");
}
@@ -148,7 +152,7 @@
Http.LOG.fine("uncompressing....");
byte[] compressed = content;
- content = GZIPUtils.unzipBestEffort(compressed, Http.MAX_CONTENT);
+ content = GZIPUtils.unzipBestEffort(compressed, http.getMaxContent());
if (content == null)
throw new HttpException("unzipBestEffort returned null");
@@ -212,9 +216,9 @@
throw new HttpException("bad content length: "+contentLengthString);
}
}
- if (Http.MAX_CONTENT >= 0
- && contentLength > Http.MAX_CONTENT) // limit download size
- contentLength = Http.MAX_CONTENT;
+ if (http.getMaxContent() >= 0
+ && contentLength > http.getMaxContent()) // limit download size
+ contentLength = http.getMaxContent();
ByteArrayOutputStream out = new ByteArrayOutputStream(Http.BUFFER_SIZE);
byte[] bytes = new byte[Http.BUFFER_SIZE];
@@ -265,8 +269,8 @@
break;
}
- if ( (contentBytesRead + chunkLen) > Http.MAX_CONTENT )
- chunkLen= Http.MAX_CONTENT - contentBytesRead;
+ if ( (contentBytesRead + chunkLen) > http.getMaxContent() )
+ chunkLen= http.getMaxContent() - contentBytesRead;
// read one chunk
int chunkBytesRead= 0;
@@ -295,7 +299,7 @@
}
if (!doneChunks) {
- if (contentBytesRead != Http.MAX_CONTENT)
+ if (contentBytesRead != http.getMaxContent())
throw new HttpException("chunk eof: !doneChunk && didn't max out");
return;
}
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java (revision 367595)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java (working copy)
@@ -46,70 +46,70 @@
public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.net.Http");
- static {
- if (NutchConf.get().getBoolean("http.verbose", false)) {
- LOG.setLevel(Level.FINE);
- } else { // shush about redirects
- Logger.getLogger("org.apache.commons.httpclient.HttpMethodDirector")
- .setLevel(Level.WARNING);
- }
- }
-
private static MultiThreadedHttpConnectionManager connectionManager =
new MultiThreadedHttpConnectionManager();
-
- private static HttpClient client;
+ // Since the NutchConf has not yet been setted,
+ // then an unconfigured client is returned.
+ private static HttpClient client = new HttpClient(connectionManager);
+
static synchronized HttpClient getClient() {
- if (client != null) return client;
- configureClient();
return client;
}
- static int MAX_THREADS_TOTAL = NutchConf.get().getInt("fetcher.threads.fetch", 10);
- static String NTLM_USERNAME = NutchConf.get().get("http.auth.ntlm.username", "");
- static String NTLM_PASSWORD = NutchConf.get().get("http.auth.ntlm.password", "");
- static String NTLM_DOMAIN = NutchConf.get().get("http.auth.ntlm.domain", "");
- static String NTLM_HOST = NutchConf.get().get("http.auth.ntlm.host", "");
+ boolean verbose = false;
+ int maxThreadsTotal = 10;
+ String ntlmUsername = "";
+ String ntlmPassword = "";
+ String ntlmDomain = "";
+ String ntlmHost = "";
- static {
- LOG.info("http.auth.ntlm.username = " + NTLM_USERNAME);
- }
-
-
public Http() {
super(LOG);
}
+ public void setConf(NutchConf conf) {
+ super.setConf(conf);
+ this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
+ this.ntlmUsername = conf.get("http.auth.ntlm.username", "");
+ this.ntlmPassword = conf.get("http.auth.ntlm.password", "");
+ this.ntlmDomain = conf.get("http.auth.ntlm.domain", "");
+ this.ntlmHost = conf.get("http.auth.ntlm.host", "");
+ Level logLevel = Level.WARNING;
+ if (conf.getBoolean("http.verbose", false)) {
+ logLevel = Level.FINE;
+ }
+ LOG.setLevel(logLevel);
+ Logger.getLogger("org.apache.commons.httpclient.HttpMethodDirector")
+ .setLevel(logLevel);
+ configureClient();
+ }
+
public static void main(String[] args) throws Exception {
main(new Http(), args);
}
protected Response getResponse(URL url, CrawlDatum datum, boolean redirect)
throws ProtocolException, IOException {
- return new HttpResponse(url, datum, redirect);
+ return new HttpResponse(this, url, datum, redirect);
}
- private static void configureClient() {
+ private void configureClient() {
- // get a client isntance -- we just need one.
-
- client = new HttpClient(connectionManager);
-
// Set up an HTTPS socket factory that accepts self-signed certs.
Protocol dummyhttps = new Protocol("https", new DummySSLProtocolSocketFactory(), 443);
Protocol.registerProtocol("https", dummyhttps);
HttpConnectionManagerParams params = connectionManager.getParams();
- params.setConnectionTimeout(TIMEOUT);
- params.setSoTimeout(TIMEOUT);
+ params.setConnectionTimeout(timeout);
+ params.setSoTimeout(timeout);
params.setSendBufferSize(BUFFER_SIZE);
params.setReceiveBufferSize(BUFFER_SIZE);
- params.setMaxTotalConnections(MAX_THREADS_TOTAL);
- if (MAX_THREADS_TOTAL > MAX_THREADS_PER_HOST) {
- params.setDefaultMaxConnectionsPerHost(MAX_THREADS_PER_HOST);
+ params.setMaxTotalConnections(maxThreadsTotal);
+ if (maxThreadsTotal > maxThreadsPerHost) {
+ params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
} else {
- params.setDefaultMaxConnectionsPerHost(MAX_THREADS_TOTAL);
+ params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
}
HostConfiguration hostConf = client.getHostConfiguration();
@@ -122,15 +122,15 @@
headers.add(new Header("Accept",
"text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
hostConf.getParams().setParameter("http.default-headers", headers);
- if (PROXY) {
- hostConf.setProxy(PROXY_HOST, PROXY_PORT);
+ if (useProxy) {
+ hostConf.setProxy(proxyHost, proxyPort);
}
- if (NTLM_USERNAME.length() > 0) {
- Credentials ntCreds = new NTCredentials(NTLM_USERNAME, NTLM_PASSWORD, NTLM_HOST, NTLM_DOMAIN);
- client.getState().setCredentials(new AuthScope(NTLM_HOST, AuthScope.ANY_PORT), ntCreds);
+ if (ntlmUsername.length() > 0) {
+ Credentials ntCreds = new NTCredentials(ntlmUsername, ntlmPassword, ntlmHost, ntlmDomain);
+ client.getState().setCredentials(new AuthScope(ntlmHost, AuthScope.ANY_PORT), ntCreds);
- LOG.info("Added NTLM credentials for " + NTLM_USERNAME);
+ LOG.info("Added NTLM credentials for " + ntlmUsername);
}
LOG.info("Configured Client");
}
-}
\ No newline at end of file
+}
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java (revision 367595)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java (working copy)
@@ -16,6 +16,7 @@
import org.apache.nutch.util.LogFormatter;
import org.apache.nutch.util.NutchConf;
+import org.apache.nutch.util.NutchConfigurable;
/**
* Implementation of RFC 2617 Basic Authentication. Usernames and passwords are stored
@@ -25,23 +26,21 @@
*
* @author Matt Tencati
*/
-public class HttpBasicAuthentication implements HttpAuthentication {
- public static final Logger LOG =
- LogFormatter.getLogger("net.nutch.net.HttpBasicAuthentication");
+public class HttpBasicAuthentication implements HttpAuthentication, NutchConfigurable {
- static {
- if (NutchConf.get().getBoolean("http.auth.verbose", false))
- LOG.setLevel(Level.FINE);
- }
+ public static final Logger LOG =
+ LogFormatter.getLogger(HttpBasicAuthentication.class.getName());
- private static Pattern basic = Pattern.compile("[bB][aA][sS][iI][cC] [rR][eE][aA][lL][mM]=\"(\\w*)\"");
+ private static Pattern basic = Pattern.compile("[bB][aA][sS][iI][cC] [rR][eE][aA][lL][mM]=\"(\\w*)\"");
private static Map authMap = new TreeMap();
-
+
+ private NutchConf conf = null;
private String challenge = null;
private ArrayList credentials = null;
private String realm = null;
+
/**
* Construct an HttpBasicAuthentication for the given challenge
* parameters. The challenge parameters are returned by the web
@@ -50,14 +49,16 @@
*
* @param challenge WWW-Authenticate header from web server
*/
- protected HttpBasicAuthentication(String challenge) throws HttpAuthenticationException {
+ protected HttpBasicAuthentication(String challenge, NutchConf conf) throws HttpAuthenticationException {
+
+ setConf(conf);
this.challenge = challenge;
LOG.fine("BasicAuthentication challenge is " + challenge);
credentials = new ArrayList();
- String username = NutchConf.get().get("http.auth.basic." + challenge + ".user");
+ String username = conf.get("http.auth.basic." + challenge + ".user");
LOG.fine("BasicAuthentication username=" + username);
- String password = NutchConf.get().get("http.auth.basic." + challenge + ".password");
+ String password = conf.get("http.auth.basic." + challenge + ".password");
LOG.fine("BasicAuthentication password=" + password);
if (username == null) {
@@ -73,6 +74,29 @@
LOG.fine("Basic credentials: " + credentials);
}
+
+ /* ---------------------------------- *
+ * *
+ * ---------------------------------- */
+
+ public void setConf(NutchConf conf) {
+ this.conf = conf;
+ if (conf.getBoolean("http.auth.verbose", false)) {
+ LOG.setLevel(Level.FINE);
+ } else {
+ LOG.setLevel(Level.WARNING);
+ }
+ }
+
+ public NutchConf getConf() {
+ return conf;
+ }
+
+ /* ---------------------------------- *
+ * *
+ * ---------------------------------- */
+
+
/**
* Gets the Basic credentials generated by this
* HttpBasicAuthentication object
@@ -105,7 +129,7 @@
* @return An HttpBasicAuthentication object or null
* if unable to generate appropriate credentials.
*/
- public static HttpBasicAuthentication getAuthentication(String challenge) {
+ public static HttpBasicAuthentication getAuthentication(String challenge, NutchConf conf) {
if (challenge == null) return null;
Matcher basicMatcher = basic.matcher(challenge);
if (basicMatcher.matches()) {
@@ -114,7 +138,7 @@
if (auth == null) {
HttpBasicAuthentication newAuth = null;
try {
- newAuth = new HttpBasicAuthentication(realm);
+ newAuth = new HttpBasicAuthentication(realm, conf);
} catch (HttpAuthenticationException hae) {
LOG.fine("HttpBasicAuthentication failed for " + challenge);
}
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java (revision 367595)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java (working copy)
@@ -15,7 +15,9 @@
import org.apache.nutch.protocol.ContentProperties;
import org.apache.nutch.util.LogFormatter;
import org.apache.nutch.util.NutchConf;
+import org.apache.nutch.util.NutchConfigurable;
+
/**
* Provides the Http protocol implementation
* with the ability to authenticate when prompted. The goal is to provide
@@ -28,25 +30,50 @@
*
* @author Matt Tencati
*/
-public class HttpAuthenticationFactory {
- /** The HTTP Authentication (WWW-Authenticate) header which is returned
+public class HttpAuthenticationFactory implements NutchConfigurable {
+
+ /**
+ * The HTTP Authentication (WWW-Authenticate) header which is returned
* by a webserver requiring authentication.
*/
public static final String AUTH_HEADER = "WWW-Authenticate";
- public static final Logger LOG =
- LogFormatter.getLogger("net.nutch.protocol.http.HttpAuthenticationFactory");
+ public static final Logger LOG =
+ LogFormatter.getLogger(HttpAuthenticationFactory.class.getName());
- static {
- if (NutchConf.get().getBoolean("http.auth.verbose", false))
- LOG.setLevel(Level.FINE);
- }
-
private static Map auths = new TreeMap();
+
+ private NutchConf conf = null;
- private HttpAuthenticationFactory() { }
- public static HttpAuthentication findAuthentication(ContentProperties header) {
+ public HttpAuthenticationFactory(NutchConf conf) {
+ setConf(conf);
+ }
+
+
+ /* ---------------------------------- *
+ * *
+ * ---------------------------------- */
+
+ public void setConf(NutchConf conf) {
+ this.conf = conf;
+ if (conf.getBoolean("http.auth.verbose", false)) {
+ LOG.setLevel(Level.FINE);
+ } else {
+ LOG.setLevel(Level.WARNING);
+ }
+ }
+
+ public NutchConf getConf() {
+ return conf;
+ }
+
+ /* ---------------------------------- *
+ * *
+ * ---------------------------------- */
+
+
+ public HttpAuthentication findAuthentication(ContentProperties header) {
if (header == null) return null;
try {
@@ -80,7 +107,7 @@
}
LOG.fine("Checking challengeString=" + challengeString);
- auth = HttpBasicAuthentication.getAuthentication(challengeString);
+ auth = HttpBasicAuthentication.getAuthentication(challengeString, conf);
if (auth != null) return auth;
//TODO Add additional Authentication lookups here
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java (revision 367595)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java (working copy)
@@ -32,6 +32,7 @@
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ContentProperties;
+import org.apache.nutch.protocol.http.api.HttpBase;
/**
@@ -47,24 +48,26 @@
private byte[] content;
+ private HttpBase http;
private int code;
private ContentProperties headers = new ContentProperties();
- public HttpResponse(URL url, CrawlDatum datum) throws IOException {
- this(url, datum, false);
+ public HttpResponse(HttpBase http, URL url, CrawlDatum datum) throws IOException {
+ this(http, url, datum, false);
}
- HttpResponse(URL url, CrawlDatum datum, boolean followRedirects) throws IOException {
+ HttpResponse(HttpBase http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {
+ this.http = http;
this.url = url;
this.base = url.toString();
this.orig = url.toString();
GetMethod get = new GetMethod(this.orig);
get.setFollowRedirects(followRedirects);
- get.setRequestHeader("User-Agent", Http.AGENT_STRING);
+ get.setRequestHeader("User-Agent", http.getUserAgent());
HttpMethodParams params = get.getParams();
// some servers cannot digest the new protocol
params.setVersion(HttpVersion.HTTP_1_0);
@@ -88,7 +91,7 @@
// for error.
try {
InputStream in = get.getResponseBodyAsStream();
- byte[] buffer = new byte[Http.BUFFER_SIZE];
+ byte[] buffer = new byte[http.BUFFER_SIZE];
int bufferFilled = 0;
int totalRead = 0;
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -147,10 +150,10 @@
private int calculateTryToRead(int totalRead) {
int tryToRead = Http.BUFFER_SIZE;
- if (Http.MAX_CONTENT <= 0) {
- return Http.BUFFER_SIZE;
- } else if (Http.MAX_CONTENT - totalRead < Http.BUFFER_SIZE) {
- tryToRead = Http.MAX_CONTENT - totalRead;
+ if (http.getMaxContent() <= 0) {
+ return http.BUFFER_SIZE;
+ } else if (http.getMaxContent() - totalRead < http.BUFFER_SIZE) {
+ tryToRead = http.getMaxContent() - totalRead;
}
return tryToRead;
}