Array
+ * of {@link UrlNormalizer}s for.
+ * @param url The url for the content that may allow us to get the type from
+ * the file suffix.
+ * @return An Array of {@link UrlNormalizer}s for the given scope.
+ * @throws PluginRuntimeException
+ */
+ public UrlNormalizer[] getURLNormalizers(String scope) {
+ List extensions = getExtensions(scope);
+ List normalizers = new Vector(extensions.size());
+
+ Iterator it = extensions.iterator();
+ while (it.hasNext()) {
+ Extension ext = (Extension) it.next();
+ UrlNormalizer normalizer = null;
+ try {
+ //check to see if we've cached this parser instance yet
+ normalizer = (UrlNormalizer) this.conf.getObject(ext.getId());
+ if (normalizer == null) {
+ // go ahead and instantiate it and then cache it
+ normalizer = (UrlNormalizer)ext.getExtensionInstance();
+ this.conf.setObject(ext.getId(), normalizer);
+ }
+ normalizers.add(normalizer);
+ } catch (PluginRuntimeException e) {
+ e.printStackTrace();
+ LOG.warning("UrlNormalizers:PluginRuntimeException when "
+ + "initializing url normalizer plugin "
+ + ext.getDescriptor().getPluginId()
+ + " instance in getURLNormalizers "
+ + "function: attempting to continue instantiating parsers");
+ }
+ }
+ return (UrlNormalizer[]) normalizers.toArray(new UrlNormalizer[normalizers.size()]);
+ }
+
+
+ /**
+ * Finds the best-suited normalizer plugin for a given scope.
+ *
+ * @param scope Scope for which we seek a normalizer plugin.
+ * @return a list of extensions to be used for this scope.
+ * If none, returns empty list.
+ * @throws PluginRuntimeException
+ */
+ private List getExtensions(String scope) {
+
+ List extensions = (List) this.conf.getObject(scope);
+
+ // Just compare the reference:
+ // if this is the empty list, we know we will find no extension.
+ if (extensions == EMPTY_EXTENSION_LIST) {
+ return EMPTY_EXTENSION_LIST;
+ }
+
+ if (extensions == null) {
+ extensions = findExtensions(scope);
+ if (extensions != null) {
+ this.conf.setObject(scope, extensions);
+ } else {
+ // Put the empty extension list into cache
+ // to remember we don't know any related extension.
+ this.conf.setObject(scope, EMPTY_EXTENSION_LIST);
+ }
+ }
+ return extensions;
+ }
+
+ /**
+ * searches a list of suitable url normalizer plugins for the given scope.
+ *
+ * @param scope Scope for which we seek a url normalizer plugin.
+ * @return List - List of extensions to be used for this scope.
+ * If none, returns null.
+ * @throws PluginRuntimeException
+ */
+ private List findExtensions(String scope) {
+
+ Extension[] extensions = this.extensionPoint.getExtensions();
+ List normalizerExtensions = new ArrayList();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ if (scope.equals(extension.getAttribute("scope")) || scope.equals(DEFAULT_PLUGIN)) {
+ normalizerExtensions.add(extension);
+ }
+ }
+ return normalizerExtensions;
+ }
+
+ public UrlNormalizer[] getNormalizers() {
+ return normalizers;
+ }
+
+ public void setNormalizers(UrlNormalizer[] normalizers) {
+ this.normalizers = normalizers;
+ }
+
+ public String normalize(String urlString) throws MalformedURLException {
+ for (int i = 0; i < this.normalizers.length; i++) {
+ if (urlString == null)
+ return null;
+ urlString = this.normalizers[i].normalize(urlString);
+ }
+ return urlString;
+ }
+}
Index: src/java/org/apache/nutch/crawl/PartitionUrlByHost.java
===================================================================
--- src/java/org/apache/nutch/crawl/PartitionUrlByHost.java (revision 396389)
+++ src/java/org/apache/nutch/crawl/PartitionUrlByHost.java (working copy)
@@ -16,18 +16,26 @@
package org.apache.nutch.crawl;
+import java.net.MalformedURLException;
import java.net.URL;
-import java.net.MalformedURLException;
-
+import java.util.logging.Logger;
+
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.LogFormatter;
+import org.apache.nutch.net.URLNormalizers;
/** Partition urls by hostname. */
public class PartitionUrlByHost implements Partitioner {
private int seed;
+ private URLNormalizers normalizers;
+ public static final String PARTITION_SCOPE = "partition";
+ public static final Logger LOG =
+ LogFormatter.getLogger("org.apache.nutch.crawl.PartitionUrlByHost");
public void configure(JobConf job) {
seed = job.getInt("partition.url.by.host.seed", 0);
+ normalizers = new URLNormalizers(job, PARTITION_SCOPE);
}
public void close() {}
@@ -36,10 +44,19 @@
public int getPartition(WritableComparable key, Writable value,
int numReduceTasks) {
String urlString = ((UTF8)key).toString();
+ if (normalizers.getNormalizers() != null) {
+ try {
+ urlString = normalizers.normalize(urlString);
+ } catch (MalformedURLException e) {
+ LOG.warning("Malformed URL Exception during PartitionURLByHost: " + e.getMessage());
+ }
+ }
+
URL url = null;
try {
url = new URL(urlString);
} catch (MalformedURLException e) {
+ LOG.warning("Malformed URL Exception during PartitionURLByHost: " + e.getMessage());
}
int hashCode = (url==null ? urlString : url.getHost()).hashCode();
@@ -48,7 +65,4 @@
return (hashCode & Integer.MAX_VALUE) % numReduceTasks;
}
-
}
-
-
Index: src/java/org/apache/nutch/crawl/Generator.java
===================================================================
--- src/java/org/apache/nutch/crawl/Generator.java (revision 396389)
+++ src/java/org/apache/nutch/crawl/Generator.java (working copy)
@@ -30,6 +30,7 @@
import org.apache.nutch.net.URLFilterException;
import org.apache.nutch.net.URLFilters;
+import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -68,6 +69,7 @@
private int maxPerHost;
private Partitioner hostPartitioner = new PartitionUrlByHost();
private URLFilters filters;
+ private URLNormalizers normalizers;
private SelectorEntry entry = new SelectorEntry();
private FloatWritable sortValue = new FloatWritable();
@@ -76,6 +78,8 @@
limit = job.getLong("crawl.topN",Long.MAX_VALUE)/job.getNumReduceTasks();
maxPerHost = job.getInt("generate.max.per.host", -1);
filters = new URLFilters(job);
+ hostPartitioner.configure(job);
+ normalizers = new URLNormalizers(job, PartitionUrlByHost.PARTITION_SCOPE);
}
public void close() {}
@@ -125,9 +129,16 @@
UTF8 url = entry.url;
if (maxPerHost > 0) { // are we counting hosts?
- String host = new URL(url.toString()).getHost();
+ String host = url.toString();
+ String oldHost = new URL(host).getHost();
+ try {
+ host = normalizers.normalize(host);
+ } catch (MalformedURLException e) {
+ LOG.warning("Malformed URL Exception during PartitionURLByHost: " + e.getMessage());
+ }
+
+ host = new URL(host).getHost();
Integer hostCount = (Integer)hostCounts.get(host);
-
// increment hostCount
hostCount = new Integer(hostCount==null ? 1 : hostCount.intValue()+1);
hostCounts.put(host, hostCount);
@@ -148,9 +159,7 @@
// maxPerHost may cause us to skip it.
count++;
}
-
}
-
}
public static class SelectorInverseMapper extends MapReduceBase implements Mapper {
Index: src/plugin/build.xml
===================================================================
--- src/plugin/build.xml (revision 396389)
+++ src/plugin/build.xml (working copy)
@@ -24,6 +24,7 @@
RegexURLNormalizer.
+ *
+ */
+public class TestRegexPartitionURLNormalizer extends RegexURLNormalizerBaseTest{
+
+ public TestRegexPartitionURLNormalizer(String testName) {
+ super(testName);
+ }
+
+ public static Test suite() {
+ return new TestSuite(TestRegexPartitionURLNormalizer.class);
+ }
+
+ public static void main(String[] args) {
+ TestRunner.run(suite());
+ }
+
+ protected UrlNormalizer getURLNormalizer(String rulesFileName) throws MalformedPatternException {
+ try {
+ return new RegexPartitionURLNormalizer(rulesFileName);
+ } catch (IOException e) {
+ fail(e.toString());
+ return null;
+ }
+ }
+
+ public void test() {
+ test("regex-url-normalize");
+ }
+}
Index: src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/urlnormalizer/regex/partition/RegexPartitionURLNormalizer.java
===================================================================
--- src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/urlnormalizer/regex/partition/RegexPartitionURLNormalizer.java (revision 0)
+++ src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/urlnormalizer/regex/partition/RegexPartitionURLNormalizer.java (revision 0)
@@ -0,0 +1,48 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.urlnormalizer.regex.partition;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.urlnormalizer.api.RegexURLNormalizerBase;
+import org.apache.oro.text.regex.MalformedPatternException;
+
+/** Allows users to do regex substitutions on all/any URLs that are encountered during partitioning.
+ *
+ * The urlnormalizer.regex.partition.file + * property should also be set to the file name of an xml file which should contain the patterns + * and substitutions to be done on encountered URLs.
+ * + */ +public class RegexPartitionURLNormalizer extends RegexURLNormalizerBase { + + /** The default constructor which is called from UrlNormalizerFactory (normalizerClass.newInstance()) in method: getNormalizer()**/ + public RegexPartitionURLNormalizer() { + super(); + } + + /** Constructor which can be passed the file name, so it doesn't look in the configuration files for it. */ + public RegexPartitionURLNormalizer(String filename)throws IOException, MalformedPatternException { + super(filename); + } + + + protected String getRulesFileName(Configuration conf) { + return conf.get("urlnormalizer.regex.partition.file"); + } +} \ No newline at end of file Index: src/plugin/urlnormalizer-regex/plugin.xml =================================================================== --- src/plugin/urlnormalizer-regex/plugin.xml (revision 0) +++ src/plugin/urlnormalizer-regex/plugin.xml (revision 0) @@ -0,0 +1,28 @@ + +RegexURLNormalizerBase.
+ *
+ */
+public abstract class RegexURLNormalizerBaseTest extends TestCase {
+
+ /** My logger */
+ protected static final Logger LOG =
+ LogFormatter.getLogger(RegexURLNormalizerBaseTest.class.getName());
+
+ private final static String SEPARATOR = System.getProperty("file.separator");
+ private final static String SAMPLES = System.getProperty("test.data", ".");
+
+ public RegexURLNormalizerBaseTest(String testName) {
+ super(testName);
+ }
+
+ protected abstract UrlNormalizer getURLNormalizer(String filename) throws MalformedPatternException;
+
+ protected void bench(int loops, String file) {
+ try {
+ bench(loops,
+ SAMPLES + SEPARATOR + file + ".xml",
+ new FileReader(SAMPLES + SEPARATOR + file + ".urls"));
+ } catch (Exception e) {
+ fail(e.toString());
+ }
+ }
+
+ protected void bench(int loops, String rulesFileName, Reader urls) {
+ long start = System.currentTimeMillis();
+ try {
+ UrlNormalizer normalizer = getURLNormalizer(rulesFileName);
+ NormalizedURL[] expected = readURLFile(urls);
+ for (int i=0; i