Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java =================================================================== --- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java (revision 1550985) +++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java (working copy) @@ -50,11 +50,11 @@ private static Pattern basic = Pattern.compile("[bB][aA][sS][iI][cC] [rR][eE][aA][lL][mM]=\"(\\w*)\""); - private static Map authMap = new TreeMap(); + private static Map authMap = new TreeMap(); private Configuration conf = null; private String challenge = null; - private ArrayList credentials = null; + private ArrayList credentials = null; private String realm = null; @@ -70,7 +70,7 @@ setConf(conf); this.challenge = challenge; - credentials = new ArrayList(); + credentials = new ArrayList(); String username = this.conf.get("http.auth.basic." + challenge + ".user"); String password = this.conf.get("http.auth.basic." + challenge + ".password"); @@ -126,7 +126,7 @@ * @return Credentials in the form of Authorization: Basic <Base64 encoded userid:password> * */ - public List getCredentials() { + public List getCredentials() { return credentials; } Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthentication.java =================================================================== --- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthentication.java (revision 1550985) +++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthentication.java (working copy) @@ -33,7 +33,7 @@ * * @return The credentials value */ - public List getCredentials(); + public List getCredentials(); /** * Gets the realm used by the HttpAuthentication object during creation. Index: src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java =================================================================== --- src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java (revision 1550985) +++ src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java (working copy) @@ -19,9 +19,6 @@ // JDK imports import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; // Slf4j Logging imports import org.slf4j.Logger; @@ -40,102 +37,59 @@ * with the ability to authenticate when prompted. The goal is to provide * multiple authentication types but for now just the {@link HttpBasicAuthentication} authentication * type is provided. - * + * * @see HttpBasicAuthentication * @see Http * @see HttpResponse - * + * * @author Matt Tencati */ public class HttpAuthenticationFactory implements Configurable { - /** - * The HTTP Authentication (WWW-Authenticate) header which is returned - * by a webserver requiring authentication. - */ - public static final String WWW_AUTHENTICATE = "WWW-Authenticate"; - - public static final Logger LOG = LoggerFactory.getLogger(HttpAuthenticationFactory.class); + /** + * The HTTP Authentication (WWW-Authenticate) header which is returned + * by a webserver requiring authentication. + */ + public static final String WWW_AUTHENTICATE = "WWW-Authenticate"; - private static Map auths = new TreeMap(); + public static final Logger LOG = LoggerFactory.getLogger(HttpAuthenticationFactory.class); - private Configuration conf = null; - - - public HttpAuthenticationFactory(Configuration conf) { - setConf(conf); - } + private Configuration conf = null; - - /* ---------------------------------- * - * * - * ---------------------------------- */ + public HttpAuthenticationFactory(Configuration conf) { + setConf(conf); + } - public void setConf(Configuration conf) { - this.conf = conf; - //if (conf.getBoolean("http.auth.verbose", false)) { - // LOG.setLevel(Level.FINE); - //} else { - // LOG.setLevel(Level.WARNING); - //} - } + public void setConf(Configuration conf) { + this.conf = conf; + } + public Configuration getConf() { + return conf; + } - public Configuration getConf() { - return conf; - } - - /* ---------------------------------- * - * * - * ---------------------------------- */ + public HttpAuthentication findAuthentication(Metadata header) { + if (header == null) return null; - public HttpAuthentication findAuthentication(Metadata header) { + try { + Collection challenge = new ArrayList(); + challenge.add(header.get(WWW_AUTHENTICATE)); - if (header == null) return null; - - try { - Collection challenge = null; - if (header instanceof Metadata) { - Object o = header.get(WWW_AUTHENTICATE); - if (o instanceof Collection) { - challenge = (Collection) o; - } else { - challenge = new ArrayList(); - challenge.add(o.toString()); - } - } else { - String challengeString = header.get(WWW_AUTHENTICATE); - if (challengeString != null) { - challenge = new ArrayList(); - challenge.add(challengeString); - } - } - if (challenge == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("Authentication challenge is null"); - } - return null; - } - - Iterator i = challenge.iterator(); - HttpAuthentication auth = null; - while (i.hasNext() && auth == null) { - String challengeString = (String)i.next(); - if (challengeString.equals("NTLM")) { - challengeString="Basic realm=techweb"; - } - - if (LOG.isTraceEnabled()) { - LOG.trace("Checking challengeString=" + challengeString); - } - auth = HttpBasicAuthentication.getAuthentication(challengeString, conf); - if (auth != null) return auth; - - //TODO Add additional Authentication lookups here - } - } catch (Exception e) { - LOG.error("Error: ", e); - } - return null; + for(String challengeString: challenge) { + if (challengeString.equals("NTLM")) + challengeString="Basic realm=techweb"; + + if (LOG.isTraceEnabled()) + LOG.trace("Checking challengeString=" + challengeString); + + HttpAuthentication auth = HttpBasicAuthentication.getAuthentication(challengeString, conf); + if (auth != null) return auth; + + //TODO Add additional Authentication lookups here + } + } catch (Exception e) { + LOG.error("Error: ", e); } + return null; + } } Index: src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java =================================================================== --- src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java (revision 1550985) +++ src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java (working copy) @@ -49,7 +49,7 @@ static final Logger LOG = LoggerFactory.getLogger(CollectionManager.class); - transient Map collectionMap = new HashMap(); + transient Map collectionMap = new HashMap(); transient URL configfile; Index: src/plugin/subcollection/src/java/org/apache/nutch/collection/Subcollection.java =================================================================== --- src/plugin/subcollection/src/java/org/apache/nutch/collection/Subcollection.java (revision 1550985) +++ src/plugin/subcollection/src/java/org/apache/nutch/collection/Subcollection.java (working copy) @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; @@ -41,10 +42,9 @@ public static final String TAG_KEY="key"; public static final String TAG_ID="id"; - ArrayList blackList = new ArrayList(); + List blackList = new ArrayList(); + List whiteList = new ArrayList(); - ArrayList whiteList = new ArrayList(); - /** * SubCollection identifier */ @@ -121,7 +121,7 @@ * * @return Whitelist entries */ - public ArrayList getWhiteList() { + public List getWhiteList() { return whiteList; } @@ -147,7 +147,7 @@ * @param whiteList * The whiteList to set. */ - public void setWhiteList(ArrayList whiteList) { + public void setWhiteList(ArrayList whiteList) { this.whiteList = whiteList; } @@ -165,10 +165,10 @@ */ public String filter(String urlString) { // first the blacklist - Iterator i = blackList.iterator(); + Iterator i = blackList.iterator(); while (i.hasNext()) { String row = (String) i.next(); - if (urlString.indexOf(row) != -1) + if (urlString.contains(row)) return null; } @@ -176,7 +176,7 @@ i = whiteList.iterator(); while (i.hasNext()) { String row = (String) i.next(); - if (urlString.indexOf(row) != -1) + if (urlString.contains(row)) return urlString; } return null; @@ -218,7 +218,7 @@ * @param list * @param text */ - protected void parseList(ArrayList list, String text) { + protected void parseList(List list, String text) { list.clear(); StringTokenizer st = new StringTokenizer(text, "\n\r"); Index: src/plugin/urlmeta/src/java/org/apache/nutch/scoring/urlmeta/URLMetaScoringFilter.java =================================================================== --- src/plugin/urlmeta/src/java/org/apache/nutch/scoring/urlmeta/URLMetaScoringFilter.java (revision 1550985) +++ src/plugin/urlmeta/src/java/org/apache/nutch/scoring/urlmeta/URLMetaScoringFilter.java (working copy) @@ -150,7 +150,7 @@ /** Boilerplate */ public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, - List inlinked) throws ScoringFilterException { + List inlinked) throws ScoringFilterException { return; } Index: src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java =================================================================== --- src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java (revision 1550985) +++ src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java (working copy) @@ -124,7 +124,6 @@ .getEmptyParseResult(content.getUrl(), getConf()); } - List entries = feed.getEntries(); String feedLink = feed.getLink(); try { feedLink = normalizers.normalize(feedLink, URLNormalizers.SCOPE_OUTLINK); @@ -134,9 +133,9 @@ feedLink = null; } - for (Iterator i = entries.iterator(); i.hasNext();) { - SyndEntry entry = (SyndEntry) i.next(); - addToMap(parseResult, feed, feedLink, entry, content); + List entries = feed.getEntries(); + for(Object entry: entries) { + addToMap(parseResult, feed, feedLink, (SyndEntry)entry, content); } String feedDesc = stripTags(feed.getDescriptionEx()); @@ -254,11 +253,10 @@ text = description.getValue(); if (text == null) { - List contents = entry.getContents(); + List contents = entry.getContents(); StringBuilder buf = new StringBuilder(); - for (Iterator i = contents.iterator(); i.hasNext();) { - SyndContent syndContent = (SyndContent) i.next(); - buf.append(syndContent.getValue()); + for (Object syndContent: contents) { + buf.append(((SyndContent)syndContent).getValue()); } text = buf.toString(); } @@ -304,7 +302,7 @@ private void addFields(Metadata parseMeta, Metadata contentMeta, SyndFeed feed, SyndEntry entry) { - List authors = entry.getAuthors(), categories = entry.getCategories(); + List authors = entry.getAuthors(), categories = entry.getCategories(); Date published = entry.getPublishedDate(), updated = entry.getUpdatedDate(); String contentType = null; @@ -325,8 +323,8 @@ } } - for (Iterator i = categories.iterator(); i.hasNext();) { - parseMeta.add(Feed.FEED_TAGS, ((SyndCategory) i.next()).getName()); + for (Object i: categories) { + parseMeta.add(Feed.FEED_TAGS, ((SyndCategory) i).getName()); } if (published != null) { @@ -341,7 +339,7 @@ contentType = description.getType(); } else { // TODO: What to do if contents.size() > 1? - List contents = entry.getContents(); + List contents = entry.getContents(); if (contents.size() > 0) { contentType = ((SyndContent) contents.get(0)).getType(); } Index: src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java =================================================================== --- src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java (revision 1550985) +++ src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java (working copy) @@ -87,10 +87,10 @@ } /** Increase the score by a sum of inlinked scores. */ - public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException { + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException { float adjust = 0.0f; for (int i = 0; i < inlinked.size(); i++) { - CrawlDatum linked = (CrawlDatum)inlinked.get(i); + CrawlDatum linked = inlinked.get(i); adjust += linked.getScore(); } if (old == null) old = datum; Index: src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java (revision 1550985) +++ src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java (working copy) @@ -24,33 +24,12 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.Reader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.URLNormalizers; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - import org.apache.nutch.util.NutchConfiguration; import junit.framework.TestCase; @@ -61,7 +40,7 @@ private RegexURLNormalizer normalizer; private Configuration conf; - private HashMap testData = new HashMap(); + private Map testData = new HashMap(); // This system property is defined in ./src/plugin/build-plugin.xml private String sampleDir = System.getProperty("test.data", "."); @@ -100,9 +79,9 @@ } public void testNormalizerScope() throws Exception { - Iterator it = testData.keySet().iterator(); + Iterator it = testData.keySet().iterator(); while (it.hasNext()) { - String scope = (String)it.next(); + String scope = it.next(); normalizeTest((NormalizedURL[])testData.get(scope), scope); } } @@ -146,7 +125,7 @@ private NormalizedURL[] readTestFile(String scope) throws IOException { File f = new File(sampleDir, "regex-normalize-" + scope + ".test"); BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8")); - List list = new ArrayList(); + List list = new ArrayList(); String line; while((line = in.readLine()) != null) { if ( line.trim().length() == 0 ||