Index: src/test/org/apache/nutch/util/TestURLUtil.java =================================================================== --- src/test/org/apache/nutch/util/TestURLUtil.java (revision 0) +++ src/test/org/apache/nutch/util/TestURLUtil.java (revision 0) @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util; + +import java.net.URL; + +import junit.framework.TestCase; + +/** Test class for URLUtil */ +public class TestURLUtil extends TestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + public void testGetDomainName() throws Exception{ + + URL url = null; + + url = new URL("http://lucene.apache.org/nutch"); + assertEquals("apache.org", URLUtil.getDomainName(url)); + + url = new URL("http://en.wikipedia.org/wiki/Java_coffee"); + assertEquals("wikipedia.org", URLUtil.getDomainName(url)); + + url = new URL("http://140.211.11.130/foundation/contributing.html"); + assertEquals("140.211.11.130", URLUtil.getDomainName(url)); + + url = new URL("http://www.example.co.uk:8080/index.html"); + assertEquals("example.co.uk", URLUtil.getDomainName(url)); + + url = new URL("http://com"); + assertEquals("com", URLUtil.getDomainName(url)); + + url = new URL("http://www.example.co.uk.com"); + assertEquals("uk.com", URLUtil.getDomainName(url)); + + //"nn" is not a tld + url = new URL("http://example.com.nn"); + assertEquals("nn", URLUtil.getDomainName(url)); + + url = new URL("http://"); + assertEquals("", URLUtil.getDomainName(url)); + + url = new URL("http://www.edu.tr.xyz"); + assertEquals("xyz", URLUtil.getDomainName(url)); + + url = new URL("http://www.example.c.se"); + assertEquals("example.c.se", URLUtil.getDomainName(url)); + + //plc.co.im is listed as a domain suffix + url = new URL("http://www.example.plc.co.im"); + assertEquals("example.plc.co.im", URLUtil.getDomainName(url)); + + //2000.hu is listed as a domain suffix + url = new URL("http://www.example.2000.hu"); + assertEquals("example.2000.hu", URLUtil.getDomainName(url)); + + //test non-ascii + url = new URL("http://www.example.商業.tw"); + assertEquals("example.商業.tw", URLUtil.getDomainName(url)); + + } + + public void testGetDomainSuffix() throws Exception{ + URL url = null; + + url = new URL("http://lucene.apache.org/nutch"); + assertEquals("org", URLUtil.getDomainSuffix(url).getDomain()); + + url = new URL("http://140.211.11.130/foundation/contributing.html"); + assertNull(URLUtil.getDomainSuffix(url)); + + url = new URL("http://www.example.co.uk:8080/index.html"); + assertEquals("co.uk", URLUtil.getDomainSuffix(url).getDomain()); + + url = new URL("http://com"); + assertEquals("com", URLUtil.getDomainSuffix(url).getDomain()); + + url = new URL("http://www.example.co.uk.com"); + assertEquals("com", URLUtil.getDomainSuffix(url).getDomain()); + + //"nn" is not a tld + url = new URL("http://example.com.nn"); + assertNull(URLUtil.getDomainSuffix(url)); + + url = new URL("http://"); + assertNull(URLUtil.getDomainSuffix(url)); + + url = new URL("http://www.edu.tr.xyz"); + assertNull(URLUtil.getDomainSuffix(url)); + + url = new URL("http://subdomain.example.edu.tr"); + assertEquals("edu.tr", URLUtil.getDomainSuffix(url).getDomain()); + + url = new URL("http://subdomain.example.presse.fr"); + assertEquals("presse.fr", URLUtil.getDomainSuffix(url).getDomain()); + + url = new URL("http://subdomain.example.presse.tr"); + assertEquals("tr", URLUtil.getDomainSuffix(url).getDomain()); + + //plc.co.im is listed as a domain suffix + url = new URL("http://www.example.plc.co.im"); + assertEquals("plc.co.im", URLUtil.getDomainSuffix(url).getDomain()); + + //2000.hu is listed as a domain suffix + url = new URL("http://www.example.2000.hu"); + assertEquals("2000.hu", URLUtil.getDomainSuffix(url).getDomain()); + + //test non-ascii + url = new URL("http://www.example.商業.tw"); + assertEquals("商業.tw", URLUtil.getDomainSuffix(url).getDomain()); + + } + + public void testGetHostSegments() throws Exception{ + URL url; + String[] segments; + + url = new URL("http://subdomain.example.edu.tr"); + segments = URLUtil.getHostSegments(url); + assertEquals("subdomain", segments[0]); + assertEquals("example", segments[1]); + assertEquals("edu", segments[2]); + assertEquals("tr", segments[3]); + + url = new URL("http://"); + segments = URLUtil.getHostSegments(url); + assertEquals(1, segments.length); + assertEquals("", segments[0]); + + url = new URL("http://140.211.11.130/foundation/contributing.html"); + segments = URLUtil.getHostSegments(url); + assertEquals(1, segments.length); + assertEquals("140.211.11.130", segments[0]); + + //test non-ascii + url = new URL("http://www.example.商業.tw"); + segments = URLUtil.getHostSegments(url); + assertEquals("www", segments[0]); + assertEquals("example", segments[1]); + assertEquals("商業", segments[2]); + assertEquals("tw", segments[3]); + + } + +} Index: src/java/org/apache/nutch/util/URLUtil.java =================================================================== --- src/java/org/apache/nutch/util/URLUtil.java (revision 0) +++ src/java/org/apache/nutch/util/URLUtil.java (revision 0) @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Pattern; + +import org.apache.nutch.util.domain.DomainSuffix; +import org.apache.nutch.util.domain.DomainSuffixes; + +/** Utility class for URL analysis */ +public class URLUtil { + + private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); + + /** Returns the domain name of the url. The domain name of a url is + * the substring of the url's hostname, w/o subdomain names. As an + * example
+ * getDomainName(conf, new URL(http://lucene.apache.org/)) + *
+ * will return
apache.org + * */ + public static String getDomainName(URL url) { + DomainSuffixes tlds = DomainSuffixes.getInstance(); + String host = url.getHost(); + //it seems that java returns hostnames ending with . + if(host.endsWith(".")) + host = host.substring(0, host.length() - 1); + if(IP_PATTERN.matcher(host).matches()) + return host; + + int index = 0; + String candidate = host; + for(;index >= 0;) { + index = candidate.indexOf('.'); + String subCandidate = candidate.substring(index+1); + if(tlds.isDomainSuffix(subCandidate)) { + return candidate; + } + candidate = subCandidate; + } + return candidate; + } + + /** Returns the domain name of the url. The domain name of a url is + * the substring of the url's hostname, w/o subdomain names. As an + * example
+ * getDomainName(conf, new http://lucene.apache.org/) + *
+ * will return
apache.org + * @throws MalformedURLException + */ + public static String getDomainName(String url) throws MalformedURLException { + return getDomainName(new URL(url)); + } + + /** Returns whether the given urls have the same domain name. + * As an example,
+ * isSameDomain(new URL("http://lucene.apache.org") + * , new URL("http://people.apache.org/")) + *
will return true. + * + * @return true if the domain names are equal + */ + public static boolean isSameDomainName(URL url1, URL url2) { + return getDomainName(url1).equalsIgnoreCase(getDomainName(url2)); + } + + /**Returns whether the given urls have the same domain name. + * As an example,
+ * isSameDomain("http://lucene.apache.org" + * ,"http://people.apache.org/") + *
will return true. + * @return true if the domain names are equal + * @throws MalformedURLException + */ + public static boolean isSameDomainName(String url1, String url2) + throws MalformedURLException { + return isSameDomainName(new URL(url1), new URL(url2)); + } + + /** Returns the {@link DomainSuffix} corresponding to the + * last public part of the hostname + */ + public static DomainSuffix getDomainSuffix(URL url) { + DomainSuffixes tlds = DomainSuffixes.getInstance(); + String host = url.getHost(); + if(IP_PATTERN.matcher(host).matches()) + return null; + + int index = 0; + String candidate = host; + for(;index >= 0;) { + index = candidate.indexOf('.'); + String subCandidate = candidate.substring(index+1); + DomainSuffix d = tlds.get(subCandidate); + if(d != null) { + return d; + } + candidate = subCandidate; + } + return null; + } + + /** Returns the {@link DomainSuffix} corresponding to the + * last public part of the hostname + */ + public static DomainSuffix getDomainSuffix(String url) throws MalformedURLException { + return getDomainSuffix(new URL(url)); + } + + /** Partitions of the hostname of the url by "." */ + public static String[] getHostSegments(URL url) { + String host = url.getHost(); + //return whole hostname, if it is an ipv4 + //TODO : handle ipv6 + if(IP_PATTERN.matcher(host).matches()) + return new String[] {host}; + return host.split("\\."); + } + + /** Partitions of the hostname of the url by "." + * @throws MalformedURLException */ + public static String[] getHostSegments(String url) throws MalformedURLException { + return getHostSegments(new URL(url)); + } + + /** For testing */ + public static void main(String[] args){ + + if(args.length!=1) { + System.err.println("Usage : URLUtil "); + return ; + } + + String url = args[0]; + try { + System.out.println(URLUtil.getDomainName(new URL(url))); + } + catch (MalformedURLException ex) { + ex.printStackTrace(); + } + } +} Index: src/java/org/apache/nutch/util/domain/TopLevelDomain.java =================================================================== --- src/java/org/apache/nutch/util/domain/TopLevelDomain.java (revision 0) +++ src/java/org/apache/nutch/util/domain/TopLevelDomain.java (revision 0) @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util.domain; + +/** + * (From wikipedia) A top-level domain (TLD) is the last part of an + * Internet domain name; that is, the letters which follow the final + * dot of any domain name. For example, in the domain name + * www.website.com, the top-level domain is com. + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + * @see http://www.iana.org/ + * @see http://en.wikipedia.org/wiki/Top-level_domain + */ +public class TopLevelDomain extends DomainSuffix { + + public enum Type { INFRASTRUCTURE, GENERIC, COUNTRY }; + + private Type type; + private String countryName = null; + + public TopLevelDomain(String domain, Type type, Status status, float boost){ + super(domain, status, boost); + this.type = type; + } + + public TopLevelDomain(String domain, Status status, float boost, String countryName){ + super(domain, status, boost); + this.type = Type.COUNTRY; + this.countryName = countryName; + } + + public Type getType() { + return type; + } + + /** Returns the country name if TLD is Country Code TLD + * @return country name or null + */ + public String getCountryName(){ + return countryName; + } + +} Index: src/java/org/apache/nutch/util/domain/DomainSuffixes.java =================================================================== --- src/java/org/apache/nutch/util/domain/DomainSuffixes.java (revision 0) +++ src/java/org/apache/nutch/util/domain/DomainSuffixes.java (revision 0) @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util.domain; + +import java.io.InputStream; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.StringUtils; + +/** + * Storage class for DomainSuffix objects + * Note: this class is singleton + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +public class DomainSuffixes { + private static final Log LOG = LogFactory.getLog(DomainSuffixes.class); + + private HashMap domains = new HashMap(); + + private static DomainSuffixes instance; + + /** private ctor */ + private DomainSuffixes() { + String file = "domain-suffixes.xml"; + InputStream input = this.getClass().getClassLoader().getResourceAsStream(file); + try { + new DomainSuffixesReader().read(this, input); + } + catch (Exception ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + + /** + * Singleton instance, lazy instantination + * @return + */ + public static DomainSuffixes getInstance() { + if(instance == null) { + instance = new DomainSuffixes(); + } + return instance; + } + + void addDomainSuffix(DomainSuffix tld) { + domains.put(tld.getDomain(), tld); + } + + /** return whether the extension is a registered domain entry */ + public boolean isDomainSuffix(String extension) { + return domains.containsKey(extension); + } + + /** + * Return the {@link DomainSuffix} object for the extension, if + * extension is a top level domain returned object will be an + * instance of {@link TopLevelDomain} + * @param extension of the domain + */ + public DomainSuffix get(String extension) { + return domains.get(extension); + } + +} Index: src/java/org/apache/nutch/util/domain/MozillaPublicSuffixListParser.java =================================================================== --- src/java/org/apache/nutch/util/domain/MozillaPublicSuffixListParser.java (revision 0) +++ src/java/org/apache/nutch/util/domain/MozillaPublicSuffixListParser.java (revision 0) @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util.domain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * Expert : This class is an utility to convert the public suffix file to xml format. + * Output file format is not valid xml, but it is a part of xml which can be + * integrated to domain-suffixes.xml file. + *
+ * Mozilla foundation maintains a list of public suffixes for the purpose of + * security enforcement to domains of cookies. The list is build by some manual + * effort. The file can be fuond at + * http://publicsuffix.org/ + * http://lxr.mozilla.org/mozilla/source/netwerk/dns/src/effective_tld_names.dat?raw=1 + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +class MozillaPublicSuffixListParser { + + void convertToXml(File input, File output) throws IOException { + + BufferedReader in = new BufferedReader(new FileReader(input)); + PrintWriter out = new PrintWriter(output); + + for(String line = in.readLine();line != null; line = in.readLine()) { + if(line.trim().length() == 0 ) { + out.println(); + continue; + } + if(line.startsWith("//")) { + //then it is a commment, convert it to xml comment + out.println(""); + } + + String[] parts = line.split("\\s"); + if(parts.length == 0 ) { + out.println(); + continue; + } + String suffix = parts[0]; + if(suffix.startsWith("!") || suffix.startsWith("*")) + suffix = suffix.substring(1); + if(suffix.startsWith(".")) + suffix = suffix.substring(1); + if(suffix.indexOf('.') == -1) { + continue;//we have the list of top-level-domains so we do not need them here. + } + suffix = suffix.toLowerCase(); + out.println(""); + } + out.close(); + in.close(); + } + + + + public static void main(String[] args ) { + if(args.length < 2) { + System.err.println("PublicSuffixLisParser "); + return; + } + try { + new MozillaPublicSuffixListParser().convertToXml(new File(args[0]), new File(args[1])); + } + catch (IOException ex) { + ex.printStackTrace(); + } + + } + +} Index: src/java/org/apache/nutch/util/domain/DomainSuffixesReader.java =================================================================== --- src/java/org/apache/nutch/util/domain/DomainSuffixesReader.java (revision 0) +++ src/java/org/apache/nutch/util/domain/DomainSuffixesReader.java (revision 0) @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util.domain; + +import java.io.IOException; +import java.io.InputStream; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.StringUtils; +import org.apache.nutch.util.domain.DomainSuffix.Status; +import org.apache.nutch.util.domain.TopLevelDomain.Type; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +/** + * For parsing xml files containing domain suffix definitions. + * Parsed xml files should validate against + * domain-suffixes.xsd + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +class DomainSuffixesReader { + + private static final Log LOG = LogFactory.getLog(DomainSuffixesReader.class); + + void read(DomainSuffixes tldEntries, InputStream input) throws IOException{ + try { + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setIgnoringComments(true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new InputSource(input)); + + Element root = document.getDocumentElement(); + + if(root != null && root.getTagName().equals("domains")) { + + Element tlds = (Element)root.getElementsByTagName("tlds").item(0); + Element suffixes = (Element)root.getElementsByTagName("suffixes").item(0); + + //read tlds + readITLDs(tldEntries, (Element)tlds.getElementsByTagName("itlds").item(0)); + readGTLDs(tldEntries, (Element)tlds.getElementsByTagName("gtlds").item(0)); + readCCTLDs(tldEntries, (Element)tlds.getElementsByTagName("cctlds").item(0)); + + readSuffixes(tldEntries, suffixes); + } + else { + throw new IOException("xml file is not valid"); + } + } + catch (ParserConfigurationException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + throw new IOException(ex.getMessage()); + } + catch (SAXException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + throw new IOException(ex.getMessage()); + } + } + + void readITLDs(DomainSuffixes tldEntries, Element el) { + NodeList children = el.getElementsByTagName("tld"); + for(int i=0;i http://lucene.apache.org/ + *
is apache.org + *
+ * This class holds three fields, + * domain field represents the suffix (such as "co.uk") + * boost is a float for boosting score of url's with this suffix + * status field represents domain's status + * + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + * @see TopLevelDomain + * @see domain-suffixes.xml + */ +public class DomainSuffix { + + /** + * Enumeration of the status of the tld. Please see domain-suffixes.xml. + */ + public enum Status { INFRASTRUCTURE, SPONSORED, UNSPONSORED + , STARTUP, PROPOSED, DELETED, PSEUDO_DOMAIN, DEPRECATED, IN_USE, NOT_IN_USE, REJECTED + }; + + private String domain; + private Status status; + private float boost; + + public static final float DEFAULT_BOOST = 1.0f; + public static final Status DEFAULT_STATUS = Status.IN_USE; + + public DomainSuffix(String domain, Status status, float boost) { + this.domain = domain; + this.status = status; + this.boost = boost; + } + + public DomainSuffix(String domain) { + this(domain, DEFAULT_STATUS, DEFAULT_BOOST); + } + + public String getDomain() { + return domain; + } + + public Status getStatus() { + return status; + } + + public float getBoost() { + return boost; + } + + @Override + public String toString() { + return domain; + } +} Index: src/java/org/apache/nutch/util/domain/package.html =================================================================== --- src/java/org/apache/nutch/util/domain/package.html (revision 0) +++ src/java/org/apache/nutch/util/domain/package.html (revision 0) @@ -0,0 +1,16 @@ + + +

org.apache.nutch.util.domain

+ +

This package contains classes for domain analysis.

+ +for information please refer to following urls : + + + + Index: src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java (revision 0) @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.scoring.tld; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collection; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.lucene.document.Document; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.protocol.Content; +import org.apache.nutch.scoring.ScoringFilter; +import org.apache.nutch.scoring.ScoringFilterException; +import org.apache.nutch.util.domain.DomainSuffix; +import org.apache.nutch.util.domain.DomainSuffixes; + + +/** + * Scoring filter to boost tlds. + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +public class TLDScoringFilter implements ScoringFilter { + + private Configuration conf; + private DomainSuffixes tldEntries; + + public TLDScoringFilter() { + tldEntries = DomainSuffixes.getInstance(); + } + + public float indexerScore(Text url, Document doc, CrawlDatum dbDatum, + CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) + throws ScoringFilterException { + + String[] tlds = doc.getValues("tld"); + float boost = 1.0f; + + if(tlds != null) { + for(String tld : tlds) { + DomainSuffix entry = tldEntries.get(tld); + if(entry != null) + boost *= entry.getBoost(); + } + } + return initScore * boost; + } + + public CrawlDatum distributeScoreToOutlink(Text fromUrl, Text toUrl, + ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount, + int validCount) throws ScoringFilterException { + return adjust; + } + + public float generatorSortValue(Text url, CrawlDatum datum, float initSort) + throws ScoringFilterException { + return initSort; + } + + public void initialScore(Text url, CrawlDatum datum) + throws ScoringFilterException { + } + + public void injectedScore(Text url, CrawlDatum datum) + throws ScoringFilterException { + } + + public void passScoreAfterParsing(Text url, Content content, Parse parse) + throws ScoringFilterException { + } + + public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content) + throws ScoringFilterException { + } + + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, + List inlinked) throws ScoringFilterException { + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + } + public CrawlDatum distributeScoreToOutlinks(Text fromUrl, ParseData parseData, + Collection> targets, CrawlDatum adjust, + int allCount) throws ScoringFilterException { + return adjust; + } + +} Index: src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html (revision 0) @@ -0,0 +1,5 @@ + + +

Top Level Domain Scoring plugin.

+ + Index: src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java (revision 0) @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.indexer.tld; + +import java.net.URL; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.indexer.IndexingException; +import org.apache.nutch.indexer.IndexingFilter; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.util.URLUtil; +import org.apache.nutch.util.domain.DomainSuffix; + +/** + * Adds the Top level domain extensions to the index + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +public class TLDIndexingFilter implements IndexingFilter { + public static final Log LOG = LogFactory.getLog(TLDIndexingFilter.class); + + private Configuration conf; + + public Document filter(Document doc, Parse parse, Text urlText, CrawlDatum datum, Inlinks inlinks) + throws IndexingException { + + try { + URL url = new URL(urlText.toString()); + DomainSuffix d = URLUtil.getDomainSuffix(url); + + // store, no index + doc.add(new Field("tld", d.getDomain(), Field.Store.YES, Field.Index.NO)); + + }catch (Exception ex) { + LOG.warn(ex); + } + + return doc; + } + + public void setConf(Configuration conf) { + this.conf = conf; + } + + public Configuration getConf() { + return this.conf; + } +} Index: src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html (revision 0) @@ -0,0 +1,5 @@ + + +

Top Level Domain Indexing plugin.

+ + Index: src/plugin/tld/plugin.xml =================================================================== --- src/plugin/tld/plugin.xml (revision 0) +++ src/plugin/tld/plugin.xml (revision 0) @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Index: src/plugin/tld/build.xml =================================================================== --- src/plugin/tld/build.xml (revision 0) +++ src/plugin/tld/build.xml (revision 0) @@ -0,0 +1,22 @@ + + + + + + + Index: src/plugin/build.xml =================================================================== --- src/plugin/build.xml (revision 560135) +++ src/plugin/build.xml (working copy) @@ -68,6 +68,7 @@ + @@ -158,6 +159,7 @@ + Index: conf/domain-suffixes.xsd =================================================================== --- conf/domain-suffixes.xsd (revision 0) +++ conf/domain-suffixes.xsd (revision 0) @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: conf/domain-suffixes.xml =================================================================== --- conf/domain-suffixes.xml (revision 0) +++ conf/domain-suffixes.xml (revision 0) @@ -0,0 +1,4354 @@ + + + + + + + + + + + + INFRASTRUCTURE + + (from http://en.wikipedia.org/wiki/.root) + vrsn-end-of-zone-marker-dummy-record.root is a domain name + listed in the DNS root zone as a diagnostic marker, whose + presence demonstrates the root zone was not truncated upon + loading by a root nameserver. It could be argued it represents + a top-level domain of .root, although technically no such + delegation exists. + + + + + INFRASTRUCTURE + + (from http://en.wikipedia.org/wiki/.arpa) .arpa is an Internet + top-level domain (TLD) used exclusively for + Internet-infrastructure purposes. It does not function as a + normal TLD where websites are registered, but rather as a + meta-TLD used to look up addresses, and for other purposes. + + + + + + + + SPONSORED + for the air transport industry + + + + UNSPONSORED + for business use + + + + SPONSORED + for Catalan language/culture + + + + UNSPONSORED + + for commercial organizations, but unrestricted + + + + + SPONSORED + for cooperatives + + + + UNSPONSORED + 1.0 + + for post-secondary educational establishments + + + + + UNSPONSORED + + for governments and their agencies in the United States + + + + + UNSPONSORED + + for informational sites, but unrestricted + + + + + UNSPONSORED + + for international organizations established by treaty + + + + + SPONSORED + for employment-related sites + + + + UNSPONSORED + for the US military + + + + SPONSORED + for sites catering to mobile devices + + + + SPONSORED + for museums + + + + UNSPONSORED + for families and individuals + + + + UNSPONSORED + + originally for network infrastructures, now unrestricted + + + + + UNSPONSORED + + originally for organizations not clearly falling within the + other gTLDs, now unrestricted + + + + + SPONSORED + for certain professions + + + + SPONSORED + + for travel agents, airlines, hoteliers, tourism bureaus, etc. + + + + + + + STARTUP + for the Asian community + + + + PROPOSED + for postal services + + + + STARTUP + + for services involving connections between the telephone + network and the Internet + + + + + PROPOSED + for geographically related sites + + + + PROPOSED + for Galicia, a country within Spain + + + + PROPOSED + for Wales, a country within the UK + + + + PROPOSED + for Scotland, a country within the UK + + + + PROPOSED + for websites designed for children + + + + PROPOSED + for websites designed for children + + + + PROPOSED + http://en.wikipedia.org/wiki/.mail + + + + PROPOSED + For Web sites of all sorts + + + + PROPOSED + For Adult entertainment sites + + + + + DELETED + + for NATO sites and operations. Replaced by .int + + + + + + PSEUDO_DOMAIN + + identifying a hostname not connected directly to the Internet, + but a bitnet network + + + + + PSEUDO_DOMAIN + + identifying a hostname not connected directly to the Internet, + but a csnet network + + + + + PSEUDO_DOMAIN + + identifying a hostname not connected directly to the Internet, + but a bitnet network + + + + + PSEUDO_DOMAIN + + .local is a pseudo top-level domain used by Apple, Inc.'s + Bonjour protocol. + + + + + PSEUDO_DOMAIN + alias of .local + + + + PSEUDO_DOMAIN + + designates an anonymous or pseudonymous address reachable via + the Tor network. + + + + + + + Ascension Island + + + + Andorra + + + + United Arab Emirates + + + + Afghanistan + + + + Antigua and Barbuda + + + + Anguilla + + + + Albania + + + + Armenia + + + + Netherlands Antilles + + + + Angola + + + + Antarctica + + + + Argentina + + + + American Samoa + + + + Austria + + + + Australia + + + + Aruba + + + + Aland Islands + + + + Azerbaijan + + + + Bosnia and Herzegovina + + + + Barbados + + + + Bangladesh + + + + Belgium + + + + Burkina Faso + + + + Bulgaria + + + + Bahrain + + + + Burundi + + + + Benin + + + + Bermuda + + + + Brunei + + + + Bolivia + + + + Brazil + + + + Bahamas + + + + Bhutan + + + + Burma + NOT_IN_USE + + not in use since re-naming of country to Myanmar, see .mm + + + + + Bouvet Island + NOT_IN_USE + not in use; no registrations + + + + Botswana + + + + Belarus + + + + Belize + + + + Canada + + + + Cocos Keeling Islands + + + + Democratic Republic of the Congo + formerly .zr - Zaire + + + + Central African Republic + + + + Republic of the Congo + + + + Switzerland + + + + Côte d'Ivoire + Ivory Coast + + + + Cook Islands + + + + Chile + + + + Cameroon + + + + People s Republic of China + + + + Colombia + + + + Costa Rica + + + + Serbia and Montenegro + DELETED + + formerly .yu - Yugoslavia; description: on June 3, 2006, + Montenegro declared independence, thus dissolving the state + union) (.cs code not assigned; no DNS) (.cs code previously + used for Czechoslovakia + + + + + Cuba + + + + Cape Verde + + + + Christmas Island + + + + Cyprus + + + + Czech Republic + + + + German Democratic Republic(East Germany) + DELETED + deleted in 1990 + + + + Germany + + + + Djibouti + + + + Denmark + + + + Dominica + + + + Dominican Republic + + + + Algeria + + + + Ecuador + + + + Estonia + + + + Egypt + + + + Western Sahara + NOT_IN_USE + not assigned; no DNS + + + + Eritrea + + + + Spain + + + + Ethiopia + + + + European Union + + code "exceptionally reserved" by ISO 3166-1 + + + + + Finland + + + + Fiji + + + + Falkland Islands + + + + Federated States of Micronesia + + + + Faroe Islands + + + + France + + + + Gabon + + + + United Kingdom + + Reserved domain by IANA; deprecated – see .uk + + + + + Grenada + + + + Georgia + + + + French Guiana + + + + Guernsey + + + + Ghana + + + + Gibraltar + + + + Greenland + + + + Gambia + + + + Guinea + + + + Guadeloupe + + + + Equatorial Guinea + + + + Greece + + + + South Georgia and the South Sandwich Islands + + + + Guatemala + + + + Guam + + + + Guinea Bissau + + + + Guyana + + + + Hong Kong + + + + Heard Island and McDonald Islands + + + + Honduras + + + + Croatia + + + + Haiti + + + + Hungary + + + + Indonesia + + + + Ireland + + + + Israel + + + + Isle of Man + + + + India + + + + British Indian Ocean Territory + + + + Iraq + + + + Iran + + + + Iceland + + + + Italy + + + + Jersey + + + + Jamaica + + + + Jordan + + + + Japan + + + + Kenya + + + + Kyrgyzstan + + + + Cambodia + + + + Kiribati + + + + Comoros + + + + Saint Kitts and Nevis + + + + North Korea + NOT_IN_USE + + not assigned; no DNS + + + + South Korea + + + + Kuwait + + + + Cayman Islands + + + + Kazakhstan + + + + Laos + + + + Lebanon + + + + Saint Lucia + + + + Liechtenstein + + + + Sri Lanka + + + + Liberia + + + + Lesotho + + + + Lithuania + + + + Luxembourg + + + + Latvia + + + + Libya + + + + Morocco + + + + Monaco + + + + Moldova + + + + Montenegro + + + + Madagascar + + + + Marshall Islands + + + + Republic of Macedonia + + + + Mali + + + + Myanmar + formerly .bu - Burma + + + + Mongolia + + + + Macau + + + + Northern Mariana Islands + + + + Martinique + + + + Mauritania + + + + Montserrat + + + + Malta + + + + Mauritius + + + + Maldives + + + + Malawi + + + + Mexico + + + + Malaysia + + + + Mozambique + + + + Namibia + + + + New Caledonia + + + + Niger + + + + Norfolk Island + + + + Nigeria + + + + Nicaragua + + + + Netherlands + + + + Norway + + + + Nepal + + + + Nauru + + + + Niue + + + + New Zealand + + + + Oman + + + + Panama + + + + Peru + + + + French Polynesia + + + + Papua New Guinea + + + + Philippines + + + + Pakistan + + + + Poland + + + + Saint Pierre and Miquelon + + + + Pitcairn Islands + + + + Puerto Rico + + + + Palestinian territories + + + + Portugal + + + + Palau + + + + Paraguay + + + + Qatar + + + + Réunion + + + + Romania + + + + Serbia + + + + Russia + + + + Rwanda + + + + Saudi Arabia + + + + Solomon Islands + + + + Seychelles + + + + Sudan + + + + Sweden + + + + Singapore + + + + Saint Helena + + + + Slovenia + + + + Svalbard and Jan Mayen Islands + NOT_IN_USE + not in use; no registrations + + + + Slovakia + + + + Sierra Leone + + + + San Marino + + + + Senegal + + + + Somalia + + + + Suriname + + + + São Tomé and Príncipe + + + + Soviet Union + DELETED + + deprecated; being phased out; code "transitionally reserved" + by ISO 3166-1 + + + + + El Salvador + + + + Syria + + + + Swaziland + + + + Turks and Caicos Islands + + + + Chad + + + + French Southern Territories + + + + Togo + + + + Thailand + + + + Tajikistan + + + + Tokelau + + + + East Timor + formerly .tp + + + + Turkmenistan + + + + Tunisia + + + + Tonga + + + + East Timor + DELETED + + deprecated - use .tl; code "transitionally reserved" by ISO + 3166-1 + + + + + Turkey + + + + Trinidad and Tobago + + + + Tuvalu + + + + Republic of China + Taiwan + + + + Tanzania + + + + Ukraine + + + + Uganda + + + + United Kingdom + + code "exceptionally reserved" by ISO 3166-1 (see also .gb) + + + + + United States Minor Outlying Islands + DELETED + see http://en.wikipedia.org/wiki/.um + + + + United States + + + + Uruguay + + + + Uzbekistan + + + + Vatican City + + + + Saint Vincent and the Grenadines + + + + Venezuela + + + + British Virgin Islands + + + + United States Virgin Islands + + + + Vietnam + + + + Vanuatu + + + + Wallis and Futuna + + + + Samoa + formerly Western Samoa + + + + Yemen + + + + Mayotte + + + + Yugoslavia + + subsequently renamed Serbia and Montenegro (code officially + replaced by .cs (see above) but still used; code + "transitionally reserved" by ISO 3166-1) + + + + + South Africa + + + + Zambia + + + + Zaire + DELETED + replaced by .cd + + + + Zimbabwe + + + + + + + + + + + + + + + + + + + + + + + + + + + + DELETED + + + DELETED + + + DELETED + + + DELETED + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +