Index: conf/top-level-domains.xsd =================================================================== --- conf/top-level-domains.xsd (revision 0) +++ conf/top-level-domains.xsd (revision 0) @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file Index: conf/top-level-domains.xml =================================================================== --- conf/top-level-domains.xml (revision 0) +++ conf/top-level-domains.xml (revision 0) @@ -0,0 +1,1933 @@ + + + + + + + + + + INFRASTRUCTURE + 1.0 + + (from http://en.wikipedia.org/wiki/.root) + vrsn-end-of-zone-marker-dummy-record.root is a domain + name listed in the DNS root zone as a diagnostic marker, + whose presence demonstrates the root zone was not + truncated upon loading by a root nameserver. It could be + argued it represents a top-level domain of .root, + although technically no such delegation exists. + + + + + INFRASTRUCTURE + 1.0 + + (from http://en.wikipedia.org/wiki/.arpa) + .arpa is an Internet top-level domain (TLD) used exclusively + for Internet-infrastructure purposes. It does not function + as a normal TLD where websites are registered, but + rather as a meta-TLD used to look up addresses, and for + other purposes. + + + + + + + + + + + SPONSORED + 1.0 + for the air transport industry + + + + UNSPONSORED + 1.0 + for business use + + + + SPONSORED + 1.0 + for Catalan language/culture + + + + UNSPONSORED + 1.0 + + for commercial organizations, but unrestricted + + + + + SPONSORED + 1.0 + for cooperatives + + + + UNSPONSORED + 1.0 + + for post-secondary educational establishments + + + + + UNSPONSORED + 1.0 + + for governments and their agencies in the United States + + + + + UNSPONSORED + 1.0 + + for informational sites, but unrestricted + + + + + UNSPONSORED + 1.0 + + for international organizations established by treaty + + + + + SPONSORED + 1.0 + for employment-related sites + + + + UNSPONSORED + 1.0 + for the US military + + + + SPONSORED + 1.0 + + for sites catering to mobile devices + + + + + SPONSORED + 1.0 + for museums + + + + UNSPONSORED + 1.0 + for families and individuals + + + + UNSPONSORED + 1.0 + + originally for network infrastructures, now unrestricted + + + + + UNSPONSORED + 1.0 + + originally for organizations not clearly falling within + the other gTLDs, now unrestricted + + + + + SPONSORED + 1.0 + for certain professions + + + + SPONSORED + 1.0 + + for travel agents, airlines, hoteliers, tourism bureaus, + etc. + + + + + + + STARTUP + 1.0 + for the Asian community + + + + PROPOSED + 1.0 + for postal services + + + + STARTUP + 1.0 + + for services involving connections between the telephone + network and the Internet + + + + + PROPOSED + 1.0 + for geographically related sites + + + + PROPOSED + 1.0 + + for Galicia, a country within Spain + + + + + PROPOSED + 1.0 + + for Wales, a country within the UK + + + + + PROPOSED + 1.0 + + for Scotland, a country within the UK + + + + + PROPOSED + 1.0 + + for websites designed for children + + + + + PROPOSED + 1.0 + + for websites designed for children + + + + + PROPOSED + 1.0 + + http://en.wikipedia.org/wiki/.mail + + + + + PROPOSED + 1.0 + For Web sites of all sorts + + + + PROPOSED + 1.0 + For Adult entertainment sites + + + + + DELETED + 1.0 + + for NATO sites and operations. Replaced by .int + + + + + + PSEUDO_DOMAIN + 1.0 + + identifying a hostname not connected directly to the + Internet, but a bitnet network + + + + + PSEUDO_DOMAIN + 1.0 + + identifying a hostname not connected directly to the + Internet, but a csnet network + + + + + PSEUDO_DOMAIN + 1.0 + + identifying a hostname not connected directly to the + Internet, but a bitnet network + + + + + PSEUDO_DOMAIN + 1.0 + + .local is a pseudo top-level domain used by Apple, + Inc.'s Bonjour protocol. + + + + + PSEUDO_DOMAIN + 1.0 + alias of .local + + + + PSEUDO_DOMAIN + 1.0 + + designates an anonymous or pseudonymous address + reachable via the Tor network. + + + + + + + + + + Ascension Island + IN_USE + 1.0 + + + + Andorra + IN_USE + 1.0 + + + + United Arab Emirates + IN_USE + 1.0 + + + + Afghanistan + IN_USE + 1.0 + + + + Antigua and Barbuda + IN_USE + 1.0 + + + + Anguilla + IN_USE + 1.0 + + + + Albania + IN_USE + 1.0 + + + + Armenia + IN_USE + 1.0 + + + + Netherlands Antilles + IN_USE + 1.0 + + + + Angola + IN_USE + 1.0 + + + + Antarctica + IN_USE + 1.0 + + + + Argentina + IN_USE + 1.0 + + + + American Samoa + IN_USE + 1.0 + + + + Austria + IN_USE + 1.0 + + + + Australia + IN_USE + 1.0 + + + + Aruba + IN_USE + 1.0 + + + + Aland Islands + IN_USE + 1.0 + + + + Azerbaijan + IN_USE + 1.0 + + + + Bosnia and Herzegovina + IN_USE + 1.0 + + + + Barbados + IN_USE + 1.0 + + + + Bangladesh + IN_USE + 1.0 + + + + Belgium + IN_USE + 1.0 + + + + Burkina Faso + IN_USE + 1.0 + + + + Bulgaria + IN_USE + 1.0 + + + + Bahrain + IN_USE + 1.0 + + + + Burundi + IN_USE + 1.0 + + + + Benin + IN_USE + 1.0 + + + + Bermuda + IN_USE + 1.0 + + + + Brunei + IN_USE + 1.0 + + + + Bolivia + IN_USE + 1.0 + + + + Brazil + IN_USE + 1.0 + + + + Bahamas + IN_USE + 1.0 + + + + Bhutan + IN_USE + 1.0 + + + + Burma + NOT_IN_USE + 1.0 + not in use since re-naming of country to Myanmar, see .mm + + + + Bouvet Island + NOT_IN_USE + 1.0 + not in use; no registrations + + + + Botswana + IN_USE + 1.0 + + + + Belarus + IN_USE + 1.0 + + + + Belize + IN_USE + 1.0 + + + + Canada + IN_USE + 1.0 + + + + Cocos Keeling Islands + IN_USE + 1.0 + + + + Democratic Republic of the Congo + IN_USE + 1.0 + formerly .zr - Zaire + + + + Central African Republic + IN_USE + 1.0 + + + + Republic of the Congo + IN_USE + 1.0 + + + + Switzerland + IN_USE + 1.0 + + + + Côte d'Ivoire + IN_USE + 1.0 + Ivory Coast + + + + Cook Islands + IN_USE + 1.0 + + + + Chile + IN_USE + 1.0 + + + + Cameroon + IN_USE + 1.0 + + + + People s Republic of China + IN_USE + 1.0 + + + + Colombia + IN_USE + 1.0 + + + + Costa Rica + IN_USE + 1.0 + + + + Serbia and Montenegro + DELETED + 1.0 + + formerly .yu - Yugoslavia; Note: on June 3, 2006, Montenegro declared + independence, thus dissolving the state union) (.cs code not assigned; no DNS) + (.cs code previously used for Czechoslovakia + + + + + Cuba + IN_USE + 1.0 + + + + Cape Verde + IN_USE + 1.0 + + + + Christmas Island + IN_USE + 1.0 + + + + Cyprus + IN_USE + 1.0 + + + + Czech Republic + IN_USE + 1.0 + + + + German Democratic Republic(East Germany) + DELETED + 1.0 + deleted in 1990 + + + + Germany + IN_USE + 1.0 + + + + Djibouti + IN_USE + 1.0 + + + + Denmark + IN_USE + 1.0 + + + + Dominica + IN_USE + 1.0 + + + + Dominican Republic + IN_USE + 1.0 + + + + Algeria + IN_USE + 1.0 + + + + Ecuador + IN_USE + 1.0 + + + + Estonia + IN_USE + 1.0 + + + + Egypt + IN_USE + 1.0 + + + + Western Sahara + NOT_IN_USE + 1.0 + not assigned; no DNS + + + + Eritrea + IN_USE + 1.0 + + + + Spain + IN_USE + 1.0 + + + + Ethiopia + IN_USE + 1.0 + + + + European Union + IN_USE + 1.0 + code "exceptionally reserved" by ISO 3166-1 + + + + Finland + IN_USE + 1.0 + + + + Fiji + IN_USE + 1.0 + + + + Falkland Islands + IN_USE + 1.0 + + + + Federated States of Micronesia + IN_USE + 1.0 + + + + Faroe Islands + IN_USE + 1.0 + + + + France + IN_USE + 1.0 + + + + Gabon + IN_USE + 1.0 + + + + United Kingdom + IN_USE + 1.0 + Reserved domain by IANA; deprecated – see .uk + + + + Grenada + IN_USE + 1.0 + + + + Georgia + IN_USE + 1.0 + + + + French Guiana + IN_USE + 1.0 + + + + Guernsey + IN_USE + 1.0 + + + + Ghana + IN_USE + 1.0 + + + + Gibraltar + IN_USE + 1.0 + + + + Greenland + IN_USE + 1.0 + + + + Gambia + IN_USE + 1.0 + + + + Guinea + IN_USE + 1.0 + + + + Guadeloupe + IN_USE + 1.0 + + + + Equatorial Guinea + IN_USE + 1.0 + + + + Greece + IN_USE + 1.0 + + + + South Georgia and the South Sandwich Islands + IN_USE + 1.0 + + + + Guatemala + IN_USE + 1.0 + + + + Guam + IN_USE + 1.0 + + + + Guinea Bissau + IN_USE + 1.0 + + + + Guyana + IN_USE + 1.0 + + + + Hong Kong + IN_USE + 1.0 + + + + Heard Island and McDonald Islands + IN_USE + 1.0 + + + + Honduras + IN_USE + 1.0 + + + + Croatia + IN_USE + 1.0 + + + + Haiti + IN_USE + 1.0 + + + + Hungary + IN_USE + 1.0 + + + + Indonesia + IN_USE + 1.0 + + + + Ireland + IN_USE + 1.0 + + + + Israel + IN_USE + 1.0 + + + + Isle of Man + IN_USE + 1.0 + + + + India + IN_USE + 1.0 + + + + British Indian Ocean Territory + IN_USE + 1.0 + + + + Iraq + IN_USE + 1.0 + + + + Iran + IN_USE + 1.0 + + + + Iceland + IN_USE + 1.0 + + + + Italy + IN_USE + 1.0 + + + + Jersey + IN_USE + 1.0 + + + + Jamaica + IN_USE + 1.0 + + + + Jordan + IN_USE + 1.0 + + + + Japan + IN_USE + 1.0 + + + + Kenya + IN_USE + 1.0 + + + + Kyrgyzstan + IN_USE + 1.0 + + + + Cambodia + IN_USE + 1.0 + + + + Kiribati + IN_USE + 1.0 + + + + Comoros + IN_USE + 1.0 + + + + Saint Kitts and Nevis + IN_USE + 1.0 + + + + North Korea + NOT_IN_USE + 1.0 + not assigned; no DNS + + + + South Korea + IN_USE + 1.0 + + + + Kuwait + IN_USE + 1.0 + + + + Cayman Islands + IN_USE + 1.0 + + + + Kazakhstan + IN_USE + 1.0 + + + + Laos + IN_USE + 1.0 + + + + Lebanon + IN_USE + 1.0 + + + + Saint Lucia + IN_USE + 1.0 + + + + Liechtenstein + IN_USE + 1.0 + + + + Sri Lanka + IN_USE + 1.0 + + + + Liberia + IN_USE + 1.0 + + + + Lesotho + IN_USE + 1.0 + + + + Lithuania + IN_USE + 1.0 + + + + Luxembourg + IN_USE + 1.0 + + + + Latvia + IN_USE + 1.0 + + + + Libya + IN_USE + 1.0 + + + + Morocco + IN_USE + 1.0 + + + + Monaco + IN_USE + 1.0 + + + + Moldova + IN_USE + 1.0 + + + + Montenegro + IN_USE + 1.0 + + + + Madagascar + IN_USE + 1.0 + + + + Marshall Islands + IN_USE + 1.0 + + + + Republic of Macedonia + IN_USE + 1.0 + + + + Mali + IN_USE + 1.0 + + + + Myanmar + IN_USE + 1.0 + formerly .bu - Burma + + + + Mongolia + IN_USE + 1.0 + + + + Macau + IN_USE + 1.0 + + + + Northern Mariana Islands + IN_USE + 1.0 + + + + Martinique + IN_USE + 1.0 + + + + Mauritania + IN_USE + 1.0 + + + + Montserrat + IN_USE + 1.0 + + + + Malta + IN_USE + 1.0 + + + + Mauritius + IN_USE + 1.0 + + + + Maldives + IN_USE + 1.0 + + + + Malawi + IN_USE + 1.0 + + + + Mexico + IN_USE + 1.0 + + + + Malaysia + IN_USE + 1.0 + + + + Mozambique + IN_USE + 1.0 + + + + Namibia + IN_USE + 1.0 + + + + New Caledonia + IN_USE + 1.0 + + + + Niger + IN_USE + 1.0 + + + + Norfolk Island + IN_USE + 1.0 + + + + Nigeria + IN_USE + 1.0 + + + + Nicaragua + IN_USE + 1.0 + + + + Netherlands + IN_USE + 1.0 + + + + Norway + IN_USE + 1.0 + + + + Nepal + IN_USE + 1.0 + + + + Nauru + IN_USE + 1.0 + + + + Niue + IN_USE + 1.0 + + + + New Zealand + IN_USE + 1.0 + + + + Oman + IN_USE + 1.0 + + + + Panama + IN_USE + 1.0 + + + + Peru + IN_USE + 1.0 + + + + French Polynesia + IN_USE + 1.0 + + + + Papua New Guinea + IN_USE + 1.0 + + + + Philippines + IN_USE + 1.0 + + + + Pakistan + IN_USE + 1.0 + + + + Poland + IN_USE + 1.0 + + + + Saint Pierre and Miquelon + IN_USE + 1.0 + + + + Pitcairn Islands + IN_USE + 1.0 + + + + Puerto Rico + IN_USE + 1.0 + + + + Palestinian territories + IN_USE + 1.0 + + + + Portugal + IN_USE + 1.0 + + + + Palau + IN_USE + 1.0 + + + + Paraguay + IN_USE + 1.0 + + + + Qatar + IN_USE + 1.0 + + + + Réunion + IN_USE + 1.0 + + + + Romania + IN_USE + 1.0 + + + + Serbia + IN_USE + 1.0 + + + + Russia + IN_USE + 1.0 + + + + Rwanda + IN_USE + 1.0 + + + + Saudi Arabia + IN_USE + 1.0 + + + + Solomon Islands + IN_USE + 1.0 + + + + Seychelles + IN_USE + 1.0 + + + + Sudan + IN_USE + 1.0 + + + + Sweden + IN_USE + 1.0 + + + + Singapore + IN_USE + 1.0 + + + + Saint Helena + IN_USE + 1.0 + + + + Slovenia + IN_USE + 1.0 + + + + Svalbard and Jan Mayen Islands + NOT_IN_USE + 1.0 + not in use; no registrations + + + + Slovakia + IN_USE + 1.0 + + + + Sierra Leone + IN_USE + 1.0 + + + + San Marino + IN_USE + 1.0 + + + + Senegal + IN_USE + 1.0 + + + + Somalia + IN_USE + 1.0 + + + + Suriname + IN_USE + 1.0 + + + + São Tomé and Príncipe + IN_USE + 1.0 + + + + Soviet Union + DELETED + 1.0 + + deprecated; being phased out; code "transitionally reserved" + by ISO 3166-1 + + + + + El Salvador + IN_USE + 1.0 + + + + Syria + IN_USE + 1.0 + + + + Swaziland + IN_USE + 1.0 + + + + Turks and Caicos Islands + IN_USE + 1.0 + + + + Chad + IN_USE + 1.0 + + + + French Southern Territories + IN_USE + 1.0 + + + + Togo + IN_USE + 1.0 + + + + Thailand + IN_USE + 1.0 + + + + Tajikistan + IN_USE + 1.0 + + + + Tokelau + IN_USE + 1.0 + + + + East Timor + IN_USE + 1.0 + formerly .tp + + + + Turkmenistan + IN_USE + 1.0 + + + + Tunisia + IN_USE + 1.0 + + + + Tonga + IN_USE + 1.0 + + + + East Timor + DELETED + 1.0 + deprecated - use .tl; code "transitionally reserved" by ISO 3166-1 + + + + Turkey + IN_USE + 1.0 + + + + Trinidad and Tobago + IN_USE + 1.0 + + + + Tuvalu + IN_USE + 1.0 + + + + Republic of China + IN_USE + 1.0 + Taiwan + + + + Tanzania + IN_USE + 1.0 + + + + Ukraine + IN_USE + 1.0 + + + + Uganda + IN_USE + 1.0 + + + + United Kingdom + IN_USE + 1.0 + code "exceptionally reserved" by ISO 3166-1 (see also .gb) + + + + United States Minor Outlying Islands + DELETED + 1.0 + see http://en.wikipedia.org/wiki/.um + + + + United States + IN_USE + 1.0 + + + + Uruguay + IN_USE + 1.0 + + + + Uzbekistan + IN_USE + 1.0 + + + + Vatican City + IN_USE + 1.0 + + + + Saint Vincent and the Grenadines + IN_USE + 1.0 + + + + Venezuela + IN_USE + 1.0 + + + + British Virgin Islands + IN_USE + 1.0 + + + + United States Virgin Islands + IN_USE + 1.0 + + + + Vietnam + IN_USE + 1.0 + + + + Vanuatu + IN_USE + 1.0 + + + + Wallis and Futuna + IN_USE + 1.0 + + + + Samoa + IN_USE + 1.0 + formerly Western Samoa + + + + Yemen + IN_USE + 1.0 + + + + Mayotte + IN_USE + 1.0 + + + + Yugoslavia + IN_USE + 1.0 + + subsequently renamed Serbia and Montenegro (code officially + replaced by .cs (see above) but still used; code + "transitionally reserved" by ISO 3166-1) + + + + + South Africa + IN_USE + 1.0 + + + + Zambia + IN_USE + 1.0 + + + + Zaire + DELETED + 1.0 + replaced by .cd + + + + Zimbabwe + IN_USE + 1.0 + + + + \ No newline at end of file Index: conf/nutch-default.xml =================================================================== --- conf/nutch-default.xml (revision 500263) +++ conf/nutch-default.xml (working copy) @@ -48,7 +48,7 @@ http.agent.name - + nutchTest HTTP 'User-Agent' request header. MUST NOT be empty - please set this to a single word uniquely related to your organization. @@ -738,7 +738,7 @@ plugin.includes - protocol-http|urlfilter-regex|parse-(text|html|js)|index-basic|query-(basic|site|url)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic) + protocol-http|urlfilter-regex|parse-(text|html|js)|index-basic|tld|query-(basic|site|url)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic) Regular expression naming plugin directory names to include. Any plugin not matching this expression is excluded. In any case you need at least include the nutch-extensionpoints plugin. By @@ -1012,4 +1012,21 @@ + + + top.level.domains.file + top-level-domains.xml + The file containing top level domain list. + + + + + top.level.domains.index + false + Whether build an index in the tld field. If true + the field is indexed with Field.Index.UN_TOKENIZED, else it + is indexed with Field.Index.NO(is not searchable) + + + Index: src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/scoring/tld/TLDScoringFilter.java (revision 0) @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.scoring.tld; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.lucene.document.Document; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.protocol.Content; +import org.apache.nutch.scoring.ScoringFilter; +import org.apache.nutch.scoring.ScoringFilterException; +import org.apache.nutch.util.tld.TLDEntries; +import org.apache.nutch.util.tld.TLDEntry; + +/** + * Scoring filter to boost tlds. + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +public class TLDScoringFilter implements ScoringFilter { + + private Configuration conf; + private TLDEntries tldEnties; + //private static final Log LOG = LogFactory.getLog(TLDScoringFilter.class); + + public float indexerScore(Text url, Document doc, CrawlDatum dbDatum, + CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) + throws ScoringFilterException { + + String[] tlds = doc.getValues("tld"); + float boost = 1.0f; + + if(tlds != null) { + for(String tld : tlds) { + TLDEntry entry = tldEnties.get(tld); + if(entry != null) + boost *= entry.getBoost(); + } + } + return initScore * boost; + } + + public CrawlDatum distributeScoreToOutlink(Text fromUrl, Text toUrl, + ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount, + int validCount) throws ScoringFilterException { + return adjust; + } + + public float generatorSortValue(Text url, CrawlDatum datum, float initSort) + throws ScoringFilterException { + return initSort; + } + + public void initialScore(Text url, CrawlDatum datum) + throws ScoringFilterException { + } + + public void injectedScore(Text url, CrawlDatum datum) + throws ScoringFilterException { + } + + public void passScoreAfterParsing(Text url, Content content, Parse parse) + throws ScoringFilterException { + } + + public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content) + throws ScoringFilterException { + } + + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, + List inlinked) throws ScoringFilterException { + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + tldEnties = TLDEntries.getInstance(conf); + } + +} Index: src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html (revision 0) @@ -0,0 +1,5 @@ + + +

Top Level Domain Scoring plugin.

+ + Index: src/plugin/tld/src/java/org/apache/nutch/util/tld/TLDEntry.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/util/tld/TLDEntry.java (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/util/tld/TLDEntry.java (revision 0) @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util.tld; + +/** + * (From wikipedia) A top-level domain (TLD) is the last part of an + * Internet domain name; that is, the letters which follow the final + * dot of any domain name. For example, in the domain name + * www.website.com, the top-level domain is com. + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + * @see http://www.iana.org/ + * @see http://en.wikipedia.org/wiki/Top-level_domain + */ +public class TLDEntry { + + /** + * Enumeration of the status of the tld. Please see topLevelDomains.xml. + */ +public enum Status { INFRASTRUCTURE, SPONSORED, UNSPONSORED + , STARTUP, PROPOSED, DELETED, PSEUDO_DOMAIN, DEPRECATED, IN_USE, NOT_IN_USE + }; + + private String domain; + private Status status; + private float boost; + + public TLDEntry(String domain, Status status, float boost){ + this.domain = domain; + this.status = status; + this.boost = boost; + } + + public static class CCTLDEntry extends TLDEntry { + private String countryName; + public CCTLDEntry(String domain, Status status, float boost, String countryName) { + super(domain, status, boost); + this.countryName = countryName; + } + public String getCountryName(){ + return countryName; + } + } + + public String getDomain() { + return domain; + } + + public Status getStatus() { + return status; + } + + public float getBoost() { + return boost; + } + +} Index: src/plugin/tld/src/java/org/apache/nutch/util/tld/TLDReader.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/util/tld/TLDReader.java (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/util/tld/TLDReader.java (revision 0) @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.util.tld; + +import java.io.IOException; +import java.io.InputStream; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.StringUtils; +import org.apache.nutch.util.tld.TLDEntry.CCTLDEntry; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + + +/** + * For parsing xml files containing tlds. + * Parsed xml files should validate against + * top-level-domains.xsd + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +class TLDReader { + + private static final Log LOG = LogFactory.getLog(TLDReader.class); + + void read(TLDEntries tldEntries, InputStream input) throws IOException{ + try { + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setIgnoringComments(true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new InputSource(input)); + + Element root = document.getDocumentElement(); + + if(root != null && root.getTagName().equals("tlds")) { + readITLDs(tldEntries, (Element)root.getElementsByTagName("itlds").item(0)); + readGTLDs(tldEntries, (Element)root.getElementsByTagName("gtlds").item(0)); + readCCTLDs(tldEntries, (Element)root.getElementsByTagName("cctlds").item(0)); + } + else { + throw new IOException("xml file is not valid"); + } + } + catch (ParserConfigurationException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + throw new IOException(ex.getMessage()); + } + catch (SAXException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + throw new IOException(ex.getMessage()); + } + + } + + void readITLDs(TLDEntries tldEntries, Element el) { + NodeList children = el.getElementsByTagName("tld"); + for(int i=0;iTLDEntrys + * Note: this class is singleton for obvious reasons + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +public class TLDEntries { + private static final Log LOG = LogFactory.getLog(TLDEntries.class); + + private HashMap itlds = new HashMap(); + private HashMap gtlds = new HashMap(); + private HashMap cctlds = new HashMap(); + + + private static TLDEntries instance; + + /** private ctor */ + private TLDEntries(Configuration conf) { + String file = conf.get("top.level.domains.file", "top-level-domains.xml"); + InputStream input = this.getClass().getClassLoader().getResourceAsStream(file); + try { + new TLDReader().read(this, input); + } + catch (Exception ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + + /** + * Singleton instance, lazy instantination + * @param conf + * @return + */ + public static TLDEntries getInstance(Configuration conf) { + if(instance == null) { + instance = new TLDEntries(conf); + } + return instance; + } + + void addITLD(TLDEntry tld) { + itlds.put(tld.getDomain(), tld); + } + + void addGTLD(TLDEntry tld) { + gtlds.put(tld.getDomain(), tld); + } + + void addCCTLD(CCTLDEntry tld) { + cctlds.put(tld.getDomain(), tld); + } + + /** return whether the extension is a itld */ + public boolean isITLD(String extension) { + return itlds.containsKey(extension); + } + + /** return whether the extension is a gtld */ + public boolean isGTLD(String extension) { + return gtlds.containsKey(extension); + } + + /** return whether the extension is a cctld */ + public boolean isCCTLD(String extension) { + return cctlds.containsKey(extension); + } + + /** return whether the extension is a itld, gtld or cctld */ + public boolean isTLD(String extension) { + return + gtlds.containsKey(extension) + || cctlds.containsKey(extension) + || itlds.containsKey(extension); + } + + /** + * Return the TLDEntry of the extension + * @param extension the tld + * @return TLDEntry if tld is either itld or gtld, + * CCTLDEntry if tld is cctld, + * null if tld does not exist + */ + public TLDEntry get(String extension) { + if(gtlds.containsKey(extension)) + return gtlds.get(extension); + else if(cctlds.containsKey(extension)) + return cctlds.get(extension); + else if(itlds.containsKey(extension)) + return itlds.get(extension); + else + return null; + } + +} Index: src/plugin/tld/src/java/org/apache/nutch/util/tld/package.html =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/util/tld/package.html (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/util/tld/package.html (revision 0) @@ -0,0 +1,6 @@ + + +

Contains classes to parse and store Top Level Domain +Information as defined by IANA.

+ + Index: src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java (revision 0) @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.indexer.tld; + +import java.net.URL; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import org.apache.nutch.parse.Parse; +import org.apache.nutch.util.tld.TLDEntries; + +import org.apache.nutch.indexer.IndexingFilter; +import org.apache.nutch.indexer.IndexingException; +import org.apache.hadoop.io.Text; + +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Inlinks; + +import org.apache.hadoop.conf.Configuration; + + +/** + * Adds the Top level domain extensions to the index + * @author Enis Soztutar <enis.soz.nutch@gmail.com> + */ +public class TLDIndexingFilter implements IndexingFilter { + public static final Log LOG = LogFactory.getLog(TLDIndexingFilter.class); + + private Configuration conf; + private TLDEntries tlds; + private boolean index = false; + + public Document filter(Document doc, Parse parse, Text urlText, CrawlDatum datum, Inlinks inlinks) + throws IndexingException { + + try { + URL url = new URL(urlText.toString()); + String host = url.getHost(); + String[] parts = host.split("\\."); + + for(int i=parts.length-1; i >= parts.length-2 && i > 0;i--) { //at most two level + if(tlds.isTLD(parts[i])) { + + if(index) { + // not store, but index, for the field to be searchable + doc.add(new Field("tld", parts[i], Field.Store.YES, Field.Index.UN_TOKENIZED)); + } + else { + // store, no index + doc.add(new Field("tld", parts[i], Field.Store.YES, Field.Index.NO)); + } + } + else + break; //tlds only appear at the end + } + + }catch (Exception ex) { + LOG.warn(ex); + } + + return doc; + } + + public void setConf(Configuration conf) { + this.conf = conf; + tlds = TLDEntries.getInstance(conf); + index = conf.getBoolean("top.level.domains.index", false); + } + + public Configuration getConf() { + return this.conf; + } +} Index: src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html =================================================================== --- src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html (revision 0) +++ src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html (revision 0) @@ -0,0 +1,5 @@ + + +

Top Level Domain Indexing plugin.

+ + Index: src/plugin/tld/plugin.xml =================================================================== --- src/plugin/tld/plugin.xml (revision 0) +++ src/plugin/tld/plugin.xml (revision 0) @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Property changes on: src/plugin/tld/plugin.xml ___________________________________________________________________ Name: svn:executable + * Index: src/plugin/tld/build.xml =================================================================== --- src/plugin/tld/build.xml (revision 0) +++ src/plugin/tld/build.xml (revision 0) @@ -0,0 +1,22 @@ + + + + + + + Property changes on: src/plugin/tld/build.xml ___________________________________________________________________ Name: svn:executable + * Index: src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java =================================================================== --- src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java (revision 500263) +++ src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java (working copy) @@ -87,7 +87,7 @@ /** Use {@link CrawlDatum#getScore()}. */ public float generatorSortValue(Text url, CrawlDatum datum, float initSort) throws ScoringFilterException { - return datum.getScore(); + return datum.getScore() * initSort; } /** Increase the score by a sum of inlinked scores. */ @@ -149,6 +149,6 @@ /** Dampen the boost value by scorePower.*/ public float indexerScore(Text url, Document doc, CrawlDatum dbDatum, CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) throws ScoringFilterException { - return (float)Math.pow(dbDatum.getScore(), scorePower); + return (float)Math.pow(dbDatum.getScore(), scorePower) * initScore; } } Index: src/plugin/build.xml =================================================================== --- src/plugin/build.xml (revision 500263) +++ src/plugin/build.xml (working copy) @@ -68,6 +68,7 @@ + @@ -157,6 +158,7 @@ +