Index: src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java =================================================================== --- src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java (revision 1333794) +++ src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java (working copy) @@ -61,7 +61,7 @@ private final static Logger LOG = LoggerFactory.getLogger(RegexURLFilterBase.class); /** An array of applicable rules */ - private RegexRule[] rules; + private List rules; /** The current configuration */ private Configuration conf; @@ -125,10 +125,10 @@ * -------------------------- */ // Inherited Javadoc - public synchronized String filter(String url) { - for (int i=0; i readRules(Reader reader) throws IOException, IllegalArgumentException { BufferedReader in = new BufferedReader(reader); - List rules = new ArrayList(); + List rules = new ArrayList(); String line; while((line=in.readLine())!=null) { @@ -205,7 +205,7 @@ RegexRule rule = createRule(sign, regex); rules.add(rule); } - return (RegexRule[]) rules.toArray(new RegexRule[rules.size()]); + return rules; } /** Index: src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexRule.java =================================================================== --- src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexRule.java (revision 1333794) +++ src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexRule.java (working copy) @@ -16,7 +16,6 @@ */ package org.apache.nutch.urlfilter.api; -import org.apache.nutch.net.*; /** @@ -26,8 +25,7 @@ */ public abstract class RegexRule { - private boolean sign; - private String regex; + private final boolean sign; /** * Constructs a new regular expression rule. @@ -41,7 +39,6 @@ */ protected RegexRule(boolean sign, String regex) { this.sign = sign; - this.regex = regex; } /** Index: src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java (revision 1333794) +++ src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java (working copy) @@ -17,36 +17,36 @@ package org.apache.nutch.net.urlnormalizer.regex; -import java.net.URL; -import java.net.MalformedURLException; -import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.io.Reader; import java.io.StringReader; - +import java.net.MalformedURLException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.List; -import java.util.ArrayList; import java.util.Iterator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import javax.xml.parsers.DocumentBuilderFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; - import org.apache.nutch.net.URLNormalizer; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; - -import javax.xml.parsers.*; -import org.w3c.dom.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; import org.xml.sax.InputSource; -import java.util.regex.*; - /** * Allows users to do regex substitutions on all/any URLs that are encountered, * which is useful for stripping session IDs from URLs. @@ -75,9 +75,20 @@ public String substitution; } - private HashMap scopedRules; + private ThreadLocal>> scopedRulesThreadLocal = + new ThreadLocal>>() { + protected java.util.HashMap> initialValue() { + return new HashMap>(); + }; + }; + + public HashMap> getScopedRules() { + return scopedRulesThreadLocal.get(); + } - private static final List EMPTY_RULES = Collections.EMPTY_LIST; + private List defaultRules; + + private static final List EMPTY_RULES = Collections.emptyList(); /** * The default constructor which is called from UrlNormalizerFactory @@ -98,45 +109,44 @@ public RegexURLNormalizer(Configuration conf, String filename) throws IOException, PatternSyntaxException { super(conf); - List rules = readConfigurationFile(filename); - if (rules != null) - scopedRules.put(URLNormalizers.SCOPE_DEFAULT, rules); + List rules = readConfigurationFile(filename); + if (rules != null) { + defaultRules = rules; + } } public void setConf(Configuration conf) { super.setConf(conf); if (conf == null) return; // the default constructor was called - if (this.scopedRules == null) { - String filename = getConf().get("urlnormalizer.regex.file"); - String stringRules = getConf().get("urlnormalizer.regex.rules"); - scopedRules = new HashMap(); - Reader reader = null; - if (stringRules != null) { - reader = new StringReader(stringRules); - } else { - reader = getConf().getConfResourceAsReader(filename); - } - List rules = null; - if (reader == null) { - LOG.warn("Can't load the default rules! "); + + String filename = getConf().get("urlnormalizer.regex.file"); + String stringRules = getConf().get("urlnormalizer.regex.rules"); + Reader reader = null; + if (stringRules != null) { + reader = new StringReader(stringRules); + } else { + reader = getConf().getConfResourceAsReader(filename); + } + List rules = null; + if (reader == null) { + LOG.warn("Can't load the default rules! "); + rules = EMPTY_RULES; + } else { + try { + rules = readConfiguration(reader); + } catch (Exception e) { + LOG.warn("Couldn't read default config: " + e); rules = EMPTY_RULES; - } else { - try { - rules = readConfiguration(reader); - } catch (Exception e) { - LOG.warn("Couldn't read default config: " + e); - rules = EMPTY_RULES; - } } - scopedRules.put(URLNormalizers.SCOPE_DEFAULT, rules); } + defaultRules = rules; } // used in JUnit test. void setConfiguration(Reader reader, String scope) { - List rules = readConfiguration(reader); - scopedRules.put(scope, rules); + List rules = readConfiguration(reader); + getScopedRules().put(scope, rules); LOG.debug("Set config for scope '" + scope + "': " + rules.size() + " rules."); } @@ -144,23 +154,20 @@ * This function does the replacements by iterating through all the regex * patterns. It accepts a string url as input and returns the altered string. */ - public synchronized String regexNormalize(String urlString, String scope) { - List curRules = (List)scopedRules.get(scope); + public String regexNormalize(String urlString, String scope) { + HashMap> scopedRules = getScopedRules(); + List curRules = scopedRules.get(scope); if (curRules == null) { // try to populate String configFile = getConf().get("urlnormalizer.regex.file." + scope); if (configFile != null) { LOG.debug("resource for scope '" + scope + "': " + configFile); - if (configFile == null) { - LOG.warn("Can't load resource for config file: " + configFile); - } else { - try { - Reader reader = getConf().getConfResourceAsReader(configFile); - curRules = readConfiguration(reader); - scopedRules.put(scope, curRules); - } catch (Exception e) { - LOG.warn("Couldn't load resource '" + configFile + "': " + e); - } + try { + Reader reader = getConf().getConfResourceAsReader(configFile); + curRules = readConfiguration(reader); + scopedRules.put(scope, curRules); + } catch (Exception e) { + LOG.warn("Couldn't load resource '" + configFile + "': " + e); } } if (curRules == EMPTY_RULES || curRules == null) { @@ -169,10 +176,9 @@ } } if (curRules == EMPTY_RULES || curRules == null) { - // use global rules - curRules = (List)scopedRules.get(URLNormalizers.SCOPE_DEFAULT); + curRules = defaultRules; } - Iterator i = curRules.iterator(); + Iterator i = curRules.iterator(); while (i.hasNext()) { Rule r = (Rule) i.next(); @@ -183,13 +189,13 @@ return urlString; } - public synchronized String normalize(String urlString, String scope) + public String normalize(String urlString, String scope) throws MalformedURLException { return regexNormalize(urlString, scope); } /** Reads the configuration file and populates a List of Rules. */ - private List readConfigurationFile(String filename) { + private List readConfigurationFile(String filename) { if (LOG.isInfoEnabled()) { LOG.info("loading " + filename); } @@ -202,8 +208,8 @@ } } - private List readConfiguration(Reader reader) { - List rules = new ArrayList(); + private List readConfiguration(Reader reader) { + List rules = new ArrayList(); try { // borrowed heavily from code in Configuration.java @@ -261,10 +267,11 @@ IOException { RegexURLNormalizer normalizer = new RegexURLNormalizer(); normalizer.setConf(NutchConfiguration.create()); - Iterator i = ((List)normalizer.scopedRules.get(URLNormalizers.SCOPE_DEFAULT)).iterator(); + HashMap> scopedRules = normalizer.getScopedRules(); + Iterator i = normalizer.defaultRules.iterator(); System.out.println("* Rules for 'DEFAULT' scope:"); while (i.hasNext()) { - Rule r = (Rule) i.next(); + Rule r = i.next(); System.out.print(" " + r.pattern.pattern() + " -> "); System.out.println(r.substitution); } @@ -272,13 +279,13 @@ if (args.length > 1) { normalizer.normalize("http://test.com", args[1]); } - if (normalizer.scopedRules.size() > 1) { - Iterator it = normalizer.scopedRules.keySet().iterator(); + if (scopedRules.size() > 1) { + Iterator it = scopedRules.keySet().iterator(); while (it.hasNext()) { - String scope = (String)it.next(); + String scope = it.next(); if (URLNormalizers.SCOPE_DEFAULT.equals(scope)) continue; System.out.println("* Rules for '" + scope + "' scope:"); - i = ((List)normalizer.scopedRules.get(scope)).iterator(); + i = ((List)scopedRules.get(scope)).iterator(); while (i.hasNext()) { Rule r = (Rule) i.next(); System.out.print(" " + r.pattern.pattern() + " -> "); Index: src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java =================================================================== --- src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java (revision 1333794) +++ src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/RegexURLFilter.java (working copy) @@ -17,15 +17,13 @@ package org.apache.nutch.urlfilter.regex; // JDK imports -import java.io.Reader; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -// Hadoop imports import org.apache.hadoop.conf.Configuration; -import org.apache.nutch.net.*; import org.apache.nutch.urlfilter.api.RegexRule; import org.apache.nutch.urlfilter.api.RegexURLFilterBase; import org.apache.nutch.util.NutchConfiguration; Index: src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java (revision 1333794) +++ src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java (working copy) @@ -28,22 +28,23 @@ import org.apache.nutch.net.URLNormalizer; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.oro.text.regex.*; /** Converts URLs to a normal form . */ -public class BasicURLNormalizer implements URLNormalizer { +public class BasicURLNormalizer extends Configured implements URLNormalizer { public static final Logger LOG = LoggerFactory.getLogger(BasicURLNormalizer.class); private Perl5Compiler compiler = new Perl5Compiler(); - private ThreadLocal matchers = new ThreadLocal() { - protected synchronized Object initialValue() { + private ThreadLocal matchers = new ThreadLocal() { + protected Perl5Matcher initialValue() { return new Perl5Matcher(); } }; - private Rule relativePathRule = null; - private Rule leadingRelativePathRule = null; - private Rule currentPathRule = null; - private Rule adjacentSlashRule = null; + private final Rule relativePathRule; + private final Rule leadingRelativePathRule; + private final Rule currentPathRule; + private final Rule adjacentSlashRule; private Configuration conf; @@ -80,7 +81,7 @@ adjacentSlashRule.substitution = new Perl5Substitution("/"); } catch (MalformedPatternException e) { - LOG.error("Failed with the following MalformedPatterException: " + e); + throw new RuntimeException(e); } } @@ -164,7 +165,7 @@ // "http://www.foo.com/../" should return a http 404 error instead of // redirecting to "http://www.foo.com". // - Perl5Matcher matcher = (Perl5Matcher)matchers.get(); + Perl5Matcher matcher = matchers.get(); while (oldLen != newLen) { // substitue first occurence of "/xx/../" by "/" @@ -204,15 +205,5 @@ public Perl5Pattern pattern; public Perl5Substitution substitution; } - - - public void setConf(Configuration conf) { - this.conf = conf; - } - - public Configuration getConf() { - return this.conf; - } - }