Index: conf/host-urlnormalizer.txt
===================================================================
--- conf/host-urlnormalizer.txt (revision 0)
+++ conf/host-urlnormalizer.txt (working copy)
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# New line separated list of hosts mapped to their desired targets.
+# wildcard hosts are supported. Format: host target
+
+# Map www.apache.org to apache.org
+www.apache.org apache.org
+
+# Map all example.org subdomains to www.example.org
+*.example2.org example2.org
Index: build.xml
===================================================================
--- build.xml (revision 1550034)
+++ build.xml (working copy)
@@ -175,11 +175,13 @@
+
@@ -587,11 +589,13 @@
+
@@ -980,6 +986,8 @@
+
+
Index: src/plugin/build.xml
===================================================================
--- src/plugin/build.xml (revision 1550034)
+++ src/plugin/build.xml (working copy)
@@ -51,11 +51,13 @@
+
+
+
+
+
+
+
+
+
+
+
Index: src/plugin/urlnormalizer-host/data/hosts.txt
===================================================================
--- src/plugin/urlnormalizer-host/data/hosts.txt (revision 0)
+++ src/plugin/urlnormalizer-host/data/hosts.txt (working copy)
@@ -0,0 +1,8 @@
+# Force all sub domains to www.
+*.example.com example.com
+
+# Force no sub domain to www. URL's
+www.example.net example.net
+
+# Force www. sub domain when hitting link without sub domain
+example.org www.example.org
\ No newline at end of file
Index: src/plugin/urlnormalizer-host/ivy.xml
===================================================================
--- src/plugin/urlnormalizer-host/ivy.xml (revision 0)
+++ src/plugin/urlnormalizer-host/ivy.xml (working copy)
@@ -0,0 +1,41 @@
+
+
+
+
+
+
+
+
+
+ Apache Nutch
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: src/plugin/urlnormalizer-host/plugin.xml
===================================================================
--- src/plugin/urlnormalizer-host/plugin.xml (revision 0)
+++ src/plugin/urlnormalizer-host/plugin.xml (working copy)
@@ -0,0 +1,43 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: src/plugin/urlnormalizer-host/src/java/org/apache/nutch/net/urlnormalizer/host/HostURLNormalizer.java
===================================================================
--- src/plugin/urlnormalizer-host/src/java/org/apache/nutch/net/urlnormalizer/host/HostURLNormalizer.java (revision 0)
+++ src/plugin/urlnormalizer-host/src/java/org/apache/nutch/net/urlnormalizer/host/HostURLNormalizer.java (working copy)
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.net.urlnormalizer.host;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLNormalizer;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.PluginRepository;
+
+/**
+ * URL normalizer for mapping hosts to their desired form. It takes
+ * a simple text file as source in the format:
+ *
+ * example.org www.example.org
+ *
+ * mapping all URL's of example.org the the www sub-domain. It also
+ * allows for wildcards to be used to map all sub-domains to another
+ * host:
+ *
+ * *.example.org www.example.org
+ */
+public class HostURLNormalizer implements URLNormalizer {
+
+ private Configuration conf;
+
+ private static final Logger LOG = LoggerFactory.getLogger(HostURLNormalizer.class);
+
+ private static String attributeFile = null;
+ private String hostsFile = null;
+ private static final HashMap hostsMap = new HashMap();
+
+ public HostURLNormalizer() {}
+
+ public HostURLNormalizer(String hostsFile) {
+ this.hostsFile = hostsFile;
+ }
+
+ private synchronized void readConfiguration(Reader configReader) throws IOException {
+ if (hostsMap.size() > 0) {
+ return;
+ }
+
+ BufferedReader reader = new BufferedReader(configReader);
+ String line, host, target;
+ int delimiterIndex;
+
+ while ((line = reader.readLine()) != null) {
+ if (StringUtils.isNotBlank(line) && !line.startsWith("#")) {
+ line.trim();
+ delimiterIndex = line.indexOf(" ");
+
+ host = line.substring(0, delimiterIndex);
+ target = line.substring(delimiterIndex + 1);
+ hostsMap.put(host, target);
+ }
+ }
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ // get the extensions for domain urlfilter
+ String pluginName = "urlnormalizer-host";
+ Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(
+ URLNormalizer.class.getName()).getExtensions();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ if (extension.getDescriptor().getPluginId().equals(pluginName)) {
+ attributeFile = extension.getAttribute("file");
+ break;
+ }
+ }
+
+ // handle blank non empty input
+ if (attributeFile != null && attributeFile.trim().equals("")) {
+ attributeFile = null;
+ }
+
+ if (attributeFile != null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Attribute \"file\" is defined for plugin " + pluginName
+ + " as " + attributeFile);
+ }
+ }
+ else {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Attribute \"file\" is not defined in plugin.xml for plugin "
+ + pluginName);
+ }
+ }
+
+ // domain file and attribute "file" take precedence if defined
+ String file = conf.get("urlnormalizer.hosts.file");
+ String stringRules = conf.get("urlnormalizer.hosts.rules");
+ if (hostsFile != null) {
+ file = hostsFile;
+ }
+ else if (attributeFile != null) {
+ file = attributeFile;
+ }
+ Reader reader = null;
+ if (stringRules != null) { // takes precedence over files
+ reader = new StringReader(stringRules);
+ } else {
+ reader = conf.getConfResourceAsReader(file);
+ }
+ try {
+ if (reader == null) {
+ reader = new FileReader(file);
+ }
+ readConfiguration(reader);
+ }
+ catch (IOException e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+
+ public String normalize(String urlString, String scope) throws MalformedURLException {
+ String host = new URL(urlString).getHost();
+
+ // Test static hosts
+ if (hostsMap.containsKey(host)) {
+ return replaceHost(urlString, host, hostsMap.get(host));
+ }
+
+ // Test for wildcard in reverse order
+ String[] hostParts = host.split("\\.");
+
+ // Use a buffer for our host parts
+ StringBuilder hostBuffer = new StringBuilder();
+
+ // This is our temp buffer keeping host parts with a wildcard
+ String wildCardHost = new String();
+
+ // Add the tld to the buffer
+ hostBuffer.append(hostParts[hostParts.length -1]);
+
+ for (int i = hostParts.length - 2; i > 0; i--) {
+ // Prepend another sub domain
+ hostBuffer.insert(0, hostParts[i] + ".");
+
+ // Make a wildcarded sub domain
+ wildCardHost = "*." + hostBuffer.toString();
+
+ // Check if this wildcard sub domain exists
+ if (hostsMap.containsKey(wildCardHost)) {
+ // Replace the original input host with the wildard replaced
+ return replaceHost(urlString, host, hostsMap.get(wildCardHost));
+ }
+ }
+
+ return urlString;
+ }
+
+ protected String replaceHost(String urlString, String host, String target) {
+ int hostIndex = urlString.indexOf(host);
+
+ StringBuilder buffer = new StringBuilder();
+
+ buffer.append(urlString.substring(0, hostIndex));
+ buffer.append(target);
+ buffer.append(urlString.substring(hostIndex + host.length()));
+
+ return buffer.toString();
+ }
+
+}
Index: src/plugin/urlnormalizer-host/src/test/org/apache/nutch/net/urlnormalizer/host/TestHostURLNormalizer.java
===================================================================
--- src/plugin/urlnormalizer-host/src/test/org/apache/nutch/net/urlnormalizer/host/TestHostURLNormalizer.java (revision 0)
+++ src/plugin/urlnormalizer-host/src/test/org/apache/nutch/net/urlnormalizer/host/TestHostURLNormalizer.java (working copy)
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.net.urlnormalizer.host;
+
+import java.net.MalformedURLException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.util.NutchConfiguration;
+
+import junit.framework.TestCase;
+
+public class TestHostURLNormalizer extends TestCase {
+
+ private final static String SEPARATOR = System.getProperty("file.separator");
+ private final static String SAMPLES = System.getProperty("test.data", ".");
+
+ public void testHostURLNormalizer() throws Exception {
+ Configuration conf = NutchConfiguration.create();
+
+ String hostsFile = SAMPLES + SEPARATOR + "hosts.txt";
+ HostURLNormalizer normalizer = new HostURLNormalizer(hostsFile);
+ normalizer.setConf(conf);
+
+ // Force www. sub domain when hitting link without sub domain
+ assertEquals("http://www.example.org/page.html", normalizer.normalize("http://example.org/page.html", URLNormalizers.SCOPE_DEFAULT));
+
+ // Force no sub domain to www. URL's
+ assertEquals("http://example.net/path/to/something.html", normalizer.normalize("http://www.example.net/path/to/something.html", URLNormalizers.SCOPE_DEFAULT));
+
+ // Force all sub domains to www.
+ assertEquals("http://example.com/?does=it&still=work", normalizer.normalize("http://example.com/?does=it&still=work", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com/buh", normalizer.normalize("http://http.www.example.com/buh", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com/blaat", normalizer.normalize("http://whatever.example.com/blaat", URLNormalizers.SCOPE_DEFAULT));
+ }
+}