Index: conf/httpclient-auth.xml
===================================================================
--- conf/httpclient-auth.xml (revision 0)
+++ conf/httpclient-auth.xml (working copy)
@@ -0,0 +1,93 @@
+
+
+
+
+
+
Index: src/plugin/protocol-httpclient/ivy.xml
===================================================================
--- src/plugin/protocol-httpclient/ivy.xml (revision 1656528)
+++ src/plugin/protocol-httpclient/ivy.xml (working copy)
@@ -36,6 +36,7 @@
+
Index: src/plugin/protocol-httpclient/plugin.xml
===================================================================
--- src/plugin/protocol-httpclient/plugin.xml (revision 1656528)
+++ src/plugin/protocol-httpclient/plugin.xml (working copy)
@@ -1,19 +1,19 @@
-
+
+
-
+
-
+
-
+ name="HttpProtocol"
+ point="org.apache.nutch.protocol.Protocol">
+
-
+ class="org.apache.nutch.protocol.httpclient.Http">
+
-
+
-
+
-
+ name="HttpsProtocol"
+ point="org.apache.nutch.protocol.Protocol">
+
-
+ class="org.apache.nutch.protocol.httpclient.Http">
+
-
+
-
+
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java (revision 1656528)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java (working copy)
@@ -21,8 +21,14 @@
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+
import org.xml.sax.SAXException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@ -45,6 +51,7 @@
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.apache.commons.httpclient.protocol.SSLProtocolSocketFactory;
+import org.apache.commons.lang.StringUtils;
// Nutch imports
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
@@ -54,10 +61,15 @@
import org.apache.nutch.util.NutchConfiguration;
/**
- * This class is a protocol plugin that configures an HTTP client for Basic,
+ * This class is a protocol plugin that configures an HTTP client for Basic,
* Digest and NTLM authentication schemes for web server as well as proxy
* server. It takes care of HTTPS protocol as well as cookies in a single fetch
- * session.
+ * session.
+ * Documentation can be found on the Nutch HttpAuthenticationSchemes
+ * wiki page.
+ * The original description of the motivation to support HttpPostAuthentication
+ * is also included on the Nutch wiki. Additionally HttpPostAuthentication development is documented
+ * at the NUTCH-827 Jira issue.
*
* @author Susam Pal
*/
@@ -85,6 +97,8 @@
private String proxyPassword;
private String proxyRealm;
+ private static HttpFormAuthConfigurer formConfigurer;
+
/**
* Returns the configured HTTP client.
*
@@ -163,7 +177,8 @@
private void configureClient() {
// Set up an HTTPS socket factory that accepts self-signed certs.
- ProtocolSocketFactory factory = new SSLProtocolSocketFactory();
+ //ProtocolSocketFactory factory = new SSLProtocolSocketFactory();
+ ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory();
Protocol https = new Protocol("https", factory, 443);
Protocol.registerProtocol("https", https);
@@ -194,9 +209,9 @@
headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
// prefer understandable formats
headers
- .add(new Header(
- "Accept",
- "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
+ .add(new Header(
+ "Accept",
+ "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
// accept gzipped content
headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
hostConf.getParams().setParameter("http.default-headers", headers);
@@ -268,6 +283,14 @@
continue;
}
+ String authMethod = credElement.getAttribute("authMethod");
+ // read http form post auth info
+ if (StringUtils.isNotBlank(authMethod)) {
+ formConfigurer = readFormAuthConfigurer(credElement,
+ authMethod);
+ continue;
+ }
+
String username = credElement.getAttribute("username");
String password = credElement.getAttribute("password");
@@ -337,6 +360,95 @@
}
/**
+ *
+ *
+ *
+ *
+ *
+ */
+ private static HttpFormAuthConfigurer readFormAuthConfigurer(
+ Element credElement, String authMethod) {
+ if ("formAuth".equals(authMethod)) {
+ HttpFormAuthConfigurer formConfigurer = new HttpFormAuthConfigurer();
+
+ String str = credElement.getAttribute("loginUrl");
+ if (StringUtils.isNotBlank(str)) {
+ formConfigurer.setLoginUrl(str.trim());
+ } else {
+ throw new IllegalArgumentException("Must set loginUrl.");
+ }
+ str = credElement.getAttribute("loginFormId");
+ if (StringUtils.isNotBlank(str)) {
+ formConfigurer.setLoginFormId(str.trim());
+ } else {
+ throw new IllegalArgumentException("Must set loginFormId.");
+ }
+ str = credElement.getAttribute("loginRedirect");
+ if (StringUtils.isNotBlank(str)) {
+ formConfigurer.setLoginRedirect(Boolean.parseBoolean(str));
+ }
+
+ NodeList nodeList = credElement.getChildNodes();
+ for (int j = 0; j < nodeList.getLength(); j++) {
+ Node node = nodeList.item(j);
+ if (!(node instanceof Element))
+ continue;
+
+ Element element = (Element) node;
+ if ("loginPostData".equals(element.getTagName())) {
+ Map loginPostData = new HashMap();
+ NodeList childNodes = element.getChildNodes();
+ for (int k = 0; k < childNodes.getLength(); k++) {
+ Node fieldNode = childNodes.item(k);
+ if (!(fieldNode instanceof Element))
+ continue;
+
+ Element fieldElement = (Element) fieldNode;
+ String name = fieldElement.getAttribute("name");
+ String value = fieldElement.getAttribute("value");
+ loginPostData.put(name, value);
+ }
+ formConfigurer.setLoginPostData(loginPostData);
+ } else if ("additionalPostHeaders".equals(element.getTagName())) {
+ Map additionalPostHeaders = new HashMap();
+ NodeList childNodes = element.getChildNodes();
+ for (int k = 0; k < childNodes.getLength(); k++) {
+ Node fieldNode = childNodes.item(k);
+ if (!(fieldNode instanceof Element))
+ continue;
+
+ Element fieldElement = (Element) fieldNode;
+ String name = fieldElement.getAttribute("name");
+ String value = fieldElement.getAttribute("value");
+ additionalPostHeaders.put(name, value);
+ }
+ formConfigurer
+ .setAdditionalPostHeaders(additionalPostHeaders);
+ } else if ("removedFormFields".equals(element.getTagName())) {
+ Set removedFormFields = new HashSet();
+ NodeList childNodes = element.getChildNodes();
+ for (int k = 0; k < childNodes.getLength(); k++) {
+ Node fieldNode = childNodes.item(k);
+ if (!(fieldNode instanceof Element))
+ continue;
+
+ Element fieldElement = (Element) fieldNode;
+ String name = fieldElement.getAttribute("name");
+ removedFormFields.add(name);
+ }
+ formConfigurer.setRemovedFormFields(removedFormFields);
+ }
+ }
+
+ return formConfigurer;
+ } else {
+ throw new IllegalArgumentException("Unsupported authMethod: "
+ + authMethod);
+ }
+ }
+
+ /**
* If credentials for the authentication scope determined from the specified
* url is not already set in the HTTP client, then this method
* sets the default credentials to fetch the specified url. If
@@ -348,6 +460,18 @@
*/
private void resolveCredentials(URL url) {
+ if (formConfigurer != null) {
+ HttpFormAuthentication formAuther = new HttpFormAuthentication(
+ formConfigurer, client, this);
+ try {
+ formAuther.login();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ return;
+ }
+
if (defaultUsername != null && defaultUsername.length() > 0) {
int port = url.getPort();
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthConfigurer.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthConfigurer.java (revision 0)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthConfigurer.java (working copy)
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.protocol.httpclient;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class HttpFormAuthConfigurer {
+ private String loginUrl;
+ private String loginFormId;
+ /**
+ * The data posted to login form, such as username(or email), password
+ */
+ private Map loginPostData;
+ /**
+ * In case we need add additional headers.
+ */
+ private Map additionalPostHeaders;
+ /**
+ * If http post login returns redirect code: 301 or 302,
+ * Http Client will automatically follow the redirect.
+ */
+ private boolean loginRedirect;
+ /**
+ * Used when we need remove some form fields.
+ */
+ private Set removedFormFields;
+
+ public HttpFormAuthConfigurer() {
+ }
+
+ public String getLoginUrl() {
+ return loginUrl;
+ }
+
+ public HttpFormAuthConfigurer setLoginUrl(String loginUrl) {
+ this.loginUrl = loginUrl;
+ return this;
+ }
+
+ public String getLoginFormId() {
+ return loginFormId;
+ }
+
+ public HttpFormAuthConfigurer setLoginFormId(String loginForm) {
+ this.loginFormId = loginForm;
+ return this;
+ }
+
+ public Map getLoginPostData() {
+ return loginPostData == null ? new HashMap()
+ : loginPostData;
+ }
+
+ public HttpFormAuthConfigurer setLoginPostData(
+ Map loginPostData) {
+ this.loginPostData = loginPostData;
+ return this;
+ }
+
+ public Map getAdditionalPostHeaders() {
+ return additionalPostHeaders == null ? new HashMap()
+ : additionalPostHeaders;
+ }
+
+ public HttpFormAuthConfigurer setAdditionalPostHeaders(
+ Map additionalPostHeaders) {
+ this.additionalPostHeaders = additionalPostHeaders;
+ return this;
+ }
+
+ public boolean isLoginRedirect() {
+ return loginRedirect;
+ }
+
+ public HttpFormAuthConfigurer setLoginRedirect(boolean redirect) {
+ this.loginRedirect = redirect;
+ return this;
+ }
+
+ public Set getRemovedFormFields() {
+ return removedFormFields == null ? new HashSet()
+ : removedFormFields;
+ }
+
+ public HttpFormAuthConfigurer setRemovedFormFields(
+ Set removedFormFields) {
+ this.removedFormFields = removedFormFields;
+ return this; }
+}
Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthentication.java
===================================================================
--- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthentication.java (revision 0)
+++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpFormAuthentication.java (working copy)
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.protocol.httpclient;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.CookieHandler;
+import java.net.CookieManager;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.NameValuePair;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.io.IOUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpFormAuthentication {
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(HttpFormAuthentication.class);
+ private static Map defaultLoginHeaders = new HashMap();
+
+ static {
+ defaultLoginHeaders.put("User-Agent", "Mozilla/5.0");
+ defaultLoginHeaders
+ .put("Accept",
+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+ defaultLoginHeaders.put("Accept-Language", "en-US,en;q=0.5");
+ defaultLoginHeaders.put("Connection", "keep-alive");
+ defaultLoginHeaders.put("Content-Type",
+ "application/x-www-form-urlencoded");
+ }
+
+ private HttpClient client;
+ private HttpFormAuthConfigurer authConfigurer = new HttpFormAuthConfigurer();
+ private String cookies;
+
+ public HttpFormAuthentication(HttpFormAuthConfigurer authConfigurer,
+ HttpClient client, Http http) {
+ this.authConfigurer = authConfigurer;
+ this.client = client;
+ defaultLoginHeaders.put("Accept", http.getAccept());
+ defaultLoginHeaders.put("Accept-Language", http.getAcceptLanguage());
+ defaultLoginHeaders.put("User-Agent", http.getUserAgent());
+ }
+
+ public HttpFormAuthentication(String loginUrl, String loginForm,
+ Map loginPostData,
+ Map additionalPostHeaders,
+ Set removedFormFields) {
+ this.authConfigurer.setLoginUrl(loginUrl);
+ this.authConfigurer.setLoginFormId(loginForm);
+ this.authConfigurer
+ .setLoginPostData(loginPostData == null ? new HashMap()
+ : loginPostData);
+ this.authConfigurer
+ .setAdditionalPostHeaders(additionalPostHeaders == null ? new HashMap()
+ : additionalPostHeaders);
+ this.authConfigurer
+ .setRemovedFormFields(removedFormFields == null ? new HashSet()
+ : removedFormFields);
+ this.client = new HttpClient();
+ }
+
+ public void login() throws Exception {
+ // make sure cookies are turned on
+ CookieHandler.setDefault(new CookieManager());
+ String pageContent = httpGetPageContent(authConfigurer.getLoginUrl());
+ List params = getLoginFormParams(pageContent);
+ sendPost(authConfigurer.getLoginUrl(), params);
+ }
+
+ private void sendPost(String url, List params)
+ throws Exception {
+ PostMethod post = null;
+ try {
+ if (authConfigurer.isLoginRedirect()) {
+ post = new PostMethod(url) {
+ @Override
+ public boolean getFollowRedirects() {
+ return true;
+ }
+ };
+ } else {
+ post = new PostMethod(url);
+ }
+ // we can't use post.setFollowRedirects(true) as it will throw
+ // IllegalArgumentException:
+ // Entity enclosing requests cannot be redirected without user
+ // intervention
+ setLoginHeader(post);
+ post.addParameters(params.toArray(new NameValuePair[0]));
+ int rspCode = client.executeMethod(post);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("rspCode: " + rspCode);
+ LOGGER.debug("\nSending 'POST' request to URL : " + url);
+
+ LOGGER.debug("Post parameters : " + params);
+ LOGGER.debug("Response Code : " + rspCode);
+ for (Header header : post.getRequestHeaders()) {
+ LOGGER.debug("Response headers : " + header);
+ }
+ }
+ String rst = IOUtils.toString(post.getResponseBodyAsStream());
+ LOGGER.debug("login post result: " + rst);
+ } finally {
+ if (post != null) {
+ post.releaseConnection();
+ }
+ }
+ }
+
+ private void setLoginHeader(PostMethod post) {
+ Map headers = new HashMap();
+ headers.putAll(defaultLoginHeaders);
+ // additionalPostHeaders can overwrite value in defaultLoginHeaders
+ headers.putAll(authConfigurer.getAdditionalPostHeaders());
+ for (Entry entry : headers.entrySet()) {
+ post.addRequestHeader(entry.getKey(), entry.getValue());
+ }
+ post.addRequestHeader("Cookie", getCookies());
+ }
+
+ private String httpGetPageContent(String url) throws IOException {
+
+ GetMethod get = new GetMethod(url);
+ try {
+ for (Entry entry : authConfigurer
+ .getAdditionalPostHeaders().entrySet()) {
+ get.addRequestHeader(entry.getKey(), entry.getValue());
+ }
+ client.executeMethod(get);
+ Header cookieHeader = get.getResponseHeader("Set-Cookie");
+ if (cookieHeader != null) {
+ setCookies(cookieHeader.getValue());
+ }
+ String rst = IOUtils.toString(get.getResponseBodyAsStream());
+ return rst;
+ } finally {
+ get.releaseConnection();
+ }
+
+ }
+
+ private List getLoginFormParams(String pageContent)
+ throws UnsupportedEncodingException {
+ List params = new ArrayList();
+ Document doc = Jsoup.parse(pageContent);
+ Element loginform = doc.getElementById(authConfigurer.getLoginFormId());
+ if (loginform == null) {
+ throw new IllegalArgumentException("No form exists: "
+ + authConfigurer.getLoginFormId());
+ }
+ Elements inputElements = loginform.getElementsByTag("input");
+ // skip fields in removedFormFields or loginPostData
+ for (Element inputElement : inputElements) {
+ String key = inputElement.attr("name");
+ String value = inputElement.attr("value");
+ if (authConfigurer.getLoginPostData().containsKey(key)
+ || authConfigurer.getRemovedFormFields().contains(key)) {
+ // value = loginPostData.get(key);
+ continue;
+ }
+ params.add(new NameValuePair(key, value));
+ }
+ // add key and value in loginPostData
+ for (Entry entry : authConfigurer.getLoginPostData()
+ .entrySet()) {
+ params.add(new NameValuePair(entry.getKey(), entry.getValue()));
+ }
+ return params;
+ }
+
+ public String getCookies() {
+ return cookies;
+ }
+
+ public void setCookies(String cookies) {
+ this.cookies = cookies;
+ }
+
+ public boolean isRedirect() {
+ return authConfigurer.isLoginRedirect();
+ }
+
+ public void setRedirect(boolean redirect) {
+ this.authConfigurer.setLoginRedirect(redirect);
+ }
+
+}