Index: src/java/org/apache/nutch/crawl/CrawlDbReducer.java =================================================================== --- src/java/org/apache/nutch/crawl/CrawlDbReducer.java (revision 5075) +++ src/java/org/apache/nutch/crawl/CrawlDbReducer.java (working copy) @@ -190,6 +190,17 @@ case CrawlDatum.STATUS_LINKED: // it was link if (oldSet) { // if old exists + // Merge metadata, the redirect could carry a cookie that allows + // passing a cookie wall. If so, the existing datum needs it or we'll + // stay stuck. Same is true for reinjecting a url with metadata that + // points to a redirect. The target needs this metadata if it doesnt + // alraedy have it. + for (Writable tkey : result.getMetaData().keySet()) { + if (!old.getMetaData().containsKey((Text)tkey)) { + old.getMetaData().put((Text)tkey, result.getMetaData().get((Text)tkey)); + } + } + result.set(old); // use it } else { result = schedule.initializeSchedule(key, result); Index: src/java/org/apache/nutch/fetcher/FetchItemQueue.java =================================================================== --- src/java/org/apache/nutch/fetcher/FetchItemQueue.java (revision 5075) +++ src/java/org/apache/nutch/fetcher/FetchItemQueue.java (working copy) @@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +47,7 @@ long minCrawlDelay; int maxThreads; Configuration conf; + Text cookie; public FetchItemQueue(Configuration conf, int maxThreads, long crawlDelay, long minCrawlDelay) { @@ -113,6 +115,14 @@ } return it; } + + public void setCookie(Text cookie) { + this.cookie = cookie; + } + + public Text getCookie() { + return cookie; + } public synchronized void dump() { LOG.info(" maxThreads = " + maxThreads); Index: src/java/org/apache/nutch/fetcher/FetcherThread.java =================================================================== --- src/java/org/apache/nutch/fetcher/FetcherThread.java (revision 5075) +++ src/java/org/apache/nutch/fetcher/FetcherThread.java (working copy) @@ -18,6 +18,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.net.HttpCookie; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; @@ -28,6 +29,7 @@ import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -94,6 +96,9 @@ private boolean ignoreExternalLinks; private String ignoreExternalLinksMode; + private static final Text COOKIE_KEY = new Text("Cookie"); + private boolean cookieEnabled; + // Used by fetcher.follow.outlinks.depth in parse private int maxOutlinksPerPage; private final int maxOutlinks; @@ -188,6 +193,7 @@ LOG.info("Using queue mode : " + queueMode); this.maxRedirect = conf.getInt("http.redirect.max", 3); + cookieEnabled = conf.getBoolean("http.enable.cookie.header", true); maxOutlinksPerPage = conf.getInt("db.max.outlinks.per.page", 100); maxOutlinks = (maxOutlinksPerPage < 0) ? Integer.MAX_VALUE : maxOutlinksPerPage; @@ -253,6 +259,15 @@ return; } } + + // Get the cookie from the queue if we can + if (cookieEnabled) { + FetchItemQueue fiq = ((FetchItemQueues)fetchQueues).getFetchItemQueue(fit.queueID); + if (fiq.getCookie() != null) { + fit.datum.getMetaData().put(COOKIE_KEY, fiq.getCookie()); + } + } + lastRequestStart.set(System.currentTimeMillis()); Text reprUrlWritable = (Text) fit.datum.getMetaData().get( Nutch.WRITABLE_REPR_URL_KEY); @@ -358,7 +373,7 @@ case ProtocolStatus.SUCCESS: // got a page pstatus = output(fit.url, fit.datum, content, status, - CrawlDatum.STATUS_FETCH_SUCCESS, fit.outlinkDepth); + CrawlDatum.STATUS_FETCH_SUCCESS, fit.outlinkDepth, fit.queueID); updateStatus(content.getContent().length); if (pstatus != null && pstatus.isSuccess() && pstatus.getMinorCode() == ParseStatus.SUCCESS_REDIRECT) { @@ -572,11 +587,23 @@ private ParseStatus output(Text key, CrawlDatum datum, Content content, ProtocolStatus pstatus, int status) { - return output(key, datum, content, pstatus, status, 0); + return output(key, datum, content, pstatus, status, 0, null); } + private ParseStatus output(FetchItem fi, Content content, + ProtocolStatus pstatus, int status) { + + return output(fi.url, fi.datum, content, pstatus, status, 0, fi.queueID); + } + + private ParseStatus output(FetchItem fi, Content content, + ProtocolStatus pstatus, int status, int outlinkDepth) { + + return output(fi.url, fi.datum, content, pstatus, status, outlinkDepth, fi.queueID); + } + private ParseStatus output(Text key, CrawlDatum datum, Content content, - ProtocolStatus pstatus, int status, int outlinkDepth) { + ProtocolStatus pstatus, int status, int outlinkDepth, String queueId) { datum.setStatus(status); datum.setFetchTime(System.currentTimeMillis()); @@ -586,6 +613,12 @@ ParseResult parseResult = null; if (content != null) { Metadata metadata = content.getMetadata(); + + // Support for cookies enabled? + if (cookieEnabled) { + // Do something then! + handleCookies(datum, content, queueId); + } // store the guessed content type in the crawldatum if (content.getContentType() != null) @@ -792,6 +825,86 @@ return null; } + private void handleCookies(CrawlDatum datum, Content content, String queueId) { + // Get any existing cookie from datum and Set-Cookie from server + String setCookie = content.getMetadata().get("Set-Cookie"); + String existingCookie = null; + + Text cookie = null; + if (datum.getMetaData().get(COOKIE_KEY) != null) { + existingCookie = datum.getMetaData().get(COOKIE_KEY).toString(); + } + + // We'll collect the final merged cookie string in here + StringBuilder cookieStr = new StringBuilder(); + + // Server wants to set a cookie? + if (setCookie != null) { + List cookies; + String parseString = "Set-Cookie2: "; + + if (setCookie != null) { + parseString += setCookie; + } + + cookies = HttpCookie.parse(parseString); + + for (int i = 0; i < cookies.size(); i++) { + StringBuilder result = new StringBuilder() + .append(cookies.get(i).getName()) + .append("=") + .append(cookies.get(i).getValue()); + + // Does this Set-Cookie overwrite any existing cookie key? + if (existingCookie != null && existingCookie.indexOf(cookies.get(i).getName()) == 0) { + // Remove that key and value from the existing cookie + existingCookie = existingCookie.replaceFirst(cookies.get(i).getName() + "=[^\\s$]+", "").trim(); + + // Null it if we got nothing left + if (existingCookie.length() == 0) { + existingCookie = null; + } + } + + cookieStr.append(result.toString()); + + // Need to append semi-colon? + if (i < cookies.size() - 1 || existingCookie != null) { + cookieStr.append("; "); + } + } + + // Do we already have a cookie? If so, merge it! + if (existingCookie != null) { + cookieStr.append(existingCookie); + } + + // Get the cookie! + cookie = new Text(cookieStr.toString().trim()); + + } else { + if (existingCookie != null) { + cookie = new Text(existingCookie); + } + } + + // + if (cookie != null) { + // Add cookie to datum so it'll reuse it later + datum.getMetaData().put(COOKIE_KEY, cookie); + + // Put the cookie in Content Metadata, otherwise it is not possible to pass + // it to our outlinks, because, unfortunately, this + // crawldatum is never passed to CookieScoringFilter.distributeScoreToOutlinks(); + content.getMetadata().remove("Set-Cookie"); // remove any potentiel cookie field first + content.getMetadata().add("Set-Cookie", cookie.toString()); + + // Add it to the fetch queue so other URL's will reuse the cookie + // Note: if more than one thread is used to fetch the host/domain! + ((FetchItemQueues)fetchQueues).getFetchItemQueue(queueId).setCookie(cookie); + } + } + private void outputRobotsTxt(List robotsTxtContent) { for (Content robotsTxt : robotsTxtContent) { LOG.debug("fetched and stored robots.txt {}", Index: src/plugin/build.xml =================================================================== --- src/plugin/build.xml (revision 5075) +++ src/plugin/build.xml (working copy) @@ -66,6 +66,7 @@ + Index: src/plugin/scoring-cookie/build.xml =================================================================== --- src/plugin/scoring-cookie/build.xml (revision 0) +++ src/plugin/scoring-cookie/build.xml (working copy) @@ -0,0 +1,27 @@ + + + + + + + + + + + + Index: src/plugin/scoring-cookie/ivy.xml =================================================================== --- src/plugin/scoring-cookie/ivy.xml (revision 0) +++ src/plugin/scoring-cookie/ivy.xml (working copy) @@ -0,0 +1,41 @@ + + + + + + + + + + Apache Nutch + + + + + + + + + + + + + + + + Index: src/plugin/scoring-cookie/plugin.xml =================================================================== --- src/plugin/scoring-cookie/plugin.xml (revision 0) +++ src/plugin/scoring-cookie/plugin.xml (working copy) @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + Index: src/plugin/scoring-cookie/src/java/org/apache/nutch/scoring/cookie/CookieScoringFilter.java =================================================================== --- src/plugin/scoring-cookie/src/java/org/apache/nutch/scoring/cookie/CookieScoringFilter.java (revision 0) +++ src/plugin/scoring-cookie/src/java/org/apache/nutch/scoring/cookie/CookieScoringFilter.java (working copy) @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.scoring.cookie; + +import java.net.HttpCookie; +import java.util.Collection; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.metadata.Nutch; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.protocol.Content; +import org.apache.nutch.scoring.ScoringFilter; +import org.apache.nutch.scoring.ScoringFilterException; +import org.apache.nutch.util.URLUtil; + +public class CookieScoringFilter implements ScoringFilter { + + private Text COOKIE_KEY = new Text("Cookie"); + private Configuration conf; + + public CookieScoringFilter() { + + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + } + + public CrawlDatum distributeScoreToOutlinks(Text fromUrl, + ParseData parseData, Collection> targets, + CrawlDatum adjust, int allCount) throws ScoringFilterException { + + try { + // Got a cookie? + if (parseData.getContentMeta().get("Set-Cookie") != null) { + // Server wants to set a cookie and we support it? + String setCookie = parseData.getContentMeta().get("Set-Cookie"); + + if (setCookie != null && setCookie.length() > 0) { + List cookies = HttpCookie.parse(setCookie.replace("Set-Cookie", "Set-Cookie2")); + + StringBuilder cookieStr = new StringBuilder(); + for (HttpCookie cookie : cookies) { + cookieStr.append(cookie); + cookieStr.append("; "); + } + + Text cookie = new Text(setCookie); + + String sourceDomain = URLUtil.getDomainName(fromUrl.toString()); + + for (Entry target : targets) { + String targetDomain = URLUtil.getDomainName(target.getKey().toString()); + if (targetDomain.equals(sourceDomain)) { + target.getValue().getMetaData().put(COOKIE_KEY, cookie); + } + } + } + } + + return adjust; + } catch (Exception e) { + throw new ScoringFilterException("Something went wrong", e); + } + } + + public float generatorSortValue(Text url, CrawlDatum datum, float initSort) + throws ScoringFilterException { + return initSort; + } + + public float indexerScore(Text url, NutchDocument doc, CrawlDatum dbDatum, + CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) + throws ScoringFilterException { + return initScore; + } + + public void initialScore(Text url, CrawlDatum datum) + throws ScoringFilterException { + // nothing to do + } + + public void injectedScore(Text url, CrawlDatum datum) + throws ScoringFilterException { + // nothing to do + } + + public void passScoreAfterParsing(Text url, Content content, Parse parse) + throws ScoringFilterException { + // nothing to do + } + + public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content) + throws ScoringFilterException { + + } + + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, + List inlinked) throws ScoringFilterException { + // nothing to do + } + +} Index: src/plugin/scoring-cookie/src/java/org/apache/nutch/scoring/cookie/package-info.java =================================================================== --- src/plugin/scoring-cookie/src/java/org/apache/nutch/scoring/cookie/package-info.java (revision 0) +++ src/plugin/scoring-cookie/src/java/org/apache/nutch/scoring/cookie/package-info.java (working copy) @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Scoring filter to pass cookies to outlinks. + */ +package org.apache.nutch.scoring.cookie; +