[-threads n] [-filtering] [-noParsing]";
if (args.length < 1) {
System.err.println(usage);
@@ -1129,7 +1170,13 @@
for (int i = 1; i < args.length; i++) { // parse command line
if (args[i].equals("-threads")) { // found -threads option
threads = Integer.parseInt(args[++i]);
- } else if (args[i].equals("-noParsing")) parsing = false;
+ }
+ else if (args[i].equals("-noParsing")) {
+ parsing = false;
+ }
+ else if (args[i].equals("-filtering")) {
+ filtering = true;
+ }
}
getConf().setInt("fetcher.threads.fetch", threads);
Index: src/plugin/build.xml
===================================================================
--- src/plugin/build.xml (revision 951473)
+++ src/plugin/build.xml (working copy)
@@ -29,6 +29,7 @@
+
@@ -130,6 +131,7 @@
+
Index: src/plugin/fetch-safe/build.xml
===================================================================
--- src/plugin/fetch-safe/build.xml (revision 0)
+++ src/plugin/fetch-safe/build.xml (revision 0)
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: src/plugin/fetch-safe/plugin.xml
===================================================================
--- src/plugin/fetch-safe/plugin.xml (revision 0)
+++ src/plugin/fetch-safe/plugin.xml (revision 0)
@@ -0,0 +1,43 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: src/plugin/fetch-safe/src/java/org/apache/nutch/fetcher/filter/safe/SafeFetchFilter.java
===================================================================
--- src/plugin/fetch-safe/src/java/org/apache/nutch/fetcher/filter/safe/SafeFetchFilter.java (revision 0)
+++ src/plugin/fetch-safe/src/java/org/apache/nutch/fetcher/filter/safe/SafeFetchFilter.java (revision 0)
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.fetcher.filter.safe;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.fetcher.FetchFilter;
+import org.apache.nutch.net.URLFilter;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseResult;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.protocol.Content;
+
+/**
+ * Filters content for bad words based on a file containing a list of bad
+ * words. Only content that doesn't contain any of the bad words is allowed.
+ *
+ * The bad words file would be setup with one word per line. If any bad word
+ * is found, the filter returns immediately.
+ *
+ * The bad words file defaults to bad-words.txt in the classpath but can be
+ * overridden using the:
+ *
+ * property "fetchfilter.badwords.file" in ./conf/nutch-*.xml,
+ * and
attribute "file" in plugin.xml of this plugin
+ *
+ * the attribute "file" has higher precedence if defined.
+ */
+public class SafeFetchFilter
+ implements FetchFilter {
+
+ private static final Log LOG = LogFactory.getLog(SafeFetchFilter.class);
+
+ // read in attribute "file" of this plugin.
+ private static String attributeFile = null;
+ private Configuration conf;
+ private String badWordsFile = null;
+ private Set badWordSet = new HashSet();
+
+ private void readBadWordsFile(Reader configReader)
+ throws IOException {
+
+ // read the configuration file, line by line
+ BufferedReader reader = new BufferedReader(configReader);
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ if (StringUtils.isNotBlank(line) && !line.startsWith("#")) {
+ // add non-blank lines and non-commented lines
+ badWordSet.add(StringUtils.lowerCase(line));
+ }
+ }
+ }
+
+ /**
+ * Default constructor.
+ */
+ public SafeFetchFilter() {
+
+ }
+
+ /**
+ * Constructor that specifies the domain file to use.
+ *
+ * @param badWordsFile The bad words file, overrides bad-words.txt default.
+ *
+ * @throws IOException
+ */
+ public SafeFetchFilter(String badWordsFile) {
+ this.badWordsFile = badWordsFile;
+ }
+
+ /**
+ * Sets the configuration.
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ // get the extensions for safe content fetch filter
+ String pluginName = "fetch-safe";
+ Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(
+ URLFilter.class.getName()).getExtensions();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ if (extension.getDescriptor().getPluginId().equals(pluginName)) {
+ attributeFile = extension.getAttribute("file");
+ break;
+ }
+ }
+
+ // handle blank non empty input
+ if (attributeFile != null && attributeFile.trim().equals("")) {
+ attributeFile = null;
+ }
+
+ if (attributeFile != null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Attribute \"file\" is defined for plugin " + pluginName
+ + " as " + attributeFile);
+ }
+ }
+ else {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Attribute \"file\" is not defined in plugin.xml for plugin "
+ + pluginName);
+ }
+ }
+
+ // domain file and attribute "file" take precedence if defined
+ String file = conf.get("fetchfilter.badwords.file");
+ if (badWordsFile != null) {
+ file = badWordsFile;
+ }
+ else if (attributeFile != null) {
+ file = attributeFile;
+ }
+
+ // get the bad words file as a classpath resource and populate the domain
+ // set with the bad words from the file.
+ try {
+ Reader reader = conf.getConfResourceAsReader(file);
+ if (reader == null) {
+ reader = new FileReader(file);
+ }
+ readBadWordsFile(reader);
+ }
+ catch (IOException e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public boolean filter(String url, Content content, ParseResult parseResult) {
+
+ try {
+
+ // turn the words on the page into a set of words in a very naive way
+ // this could be enhanced but is simply a way to demonstrate how the
+ // FetchFilter interface and plugin works
+ if (parseResult != null) {
+ for (Entry entry : parseResult) {
+ Parse parse = entry.getValue();
+ String[] words = StringUtils.split(StringUtils.lowerCase(parse.getText()));
+ Set wordSet = new HashSet();
+ wordSet.addAll(Arrays.asList(words));
+
+ for (String word : wordSet) {
+ if (badWordSet.contains(word)) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+
+ LOG.error("Could not apply filter on: " + url + "\n"
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+
+ return true;
+ }
+}
Index: src/plugin/nutch-extensionpoints/plugin.xml
===================================================================
--- src/plugin/nutch-extensionpoints/plugin.xml (revision 951473)
+++ src/plugin/nutch-extensionpoints/plugin.xml (working copy)
@@ -65,6 +65,10 @@
name="Nutch URL Normalizer"/>
+
+