Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/fetcher/Fetcher.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/fetcher/Fetcher.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/fetcher/Fetcher.java (working copy) @@ -103,8 +103,12 @@ CrawlDatum datum = new CrawlDatum(); while (true) { - if (LogFormatter.hasLoggedSevere()) // something bad happened - break; // exit + if (conf.get(NutchConfiguration.LOG_SEVERE_FIELD) != null + && conf.get(NutchConfiguration.LOG_SEVERE_FIELD) + .equals(NutchConfiguration.LOG_SEVERE)) // something + // bad + // happened + break; // exit try { // get next entry from input if (!input.next(key, datum)) { @@ -113,6 +117,7 @@ } catch (IOException e) { e.printStackTrace(); LOG.severe("fetcher caught:"+e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); break; } @@ -213,6 +218,7 @@ } catch (Throwable e) { e.printStackTrace(); LOG.severe("fetcher caught:"+e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } finally { synchronized (Fetcher.this) {activeThreads--;} // count threads } @@ -285,6 +291,7 @@ } catch (IOException e) { e.printStackTrace(); LOG.severe("fetcher caught:"+e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } if (parse != null) return parse.getData().getStatus(); else return null; Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/net/BasicUrlNormalizer.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/net/BasicUrlNormalizer.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/net/BasicUrlNormalizer.java (working copy) @@ -40,7 +40,7 @@ private Rule relativePathRule = null; private Rule leadingRelativePathRule = null; - private Configuration conf; + protected Configuration conf; public BasicUrlNormalizer() { try { Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/net/RegexUrlNormalizer.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/net/RegexUrlNormalizer.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/net/RegexUrlNormalizer.java (working copy) @@ -105,8 +105,12 @@ DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(filename); Element root = doc.getDocumentElement(); - if (!"regex-normalize".equals(root.getTagName())) - LOG.severe("bad conf file: top-level element not "); + if (!"regex-normalize".equals(root.getTagName())){ + LOG.severe("bad conf file: top-level element not "); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); + } + + NodeList regexes = root.getChildNodes(); for (int i = 0; i < regexes.getLength(); i++) { Node regexNode = regexes.item(i); @@ -140,6 +144,7 @@ } catch (Exception e) { LOG.severe("error parsing " + filename +" conf file: " + e); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } return rules; } Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/ontology/Ontology.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/ontology/Ontology.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/ontology/Ontology.java (working copy) @@ -22,8 +22,11 @@ // Nutch imports import org.apache.nutch.plugin.Pluggable; +// Hadoop imports +import org.apache.hadoop.conf.Configurable; -public interface Ontology extends Pluggable { + +public interface Ontology extends Pluggable, Configurable { /** The name of the extension point. */ public final static String X_POINT_ID = Ontology.class.getName(); Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/ontology/OntologyFactory.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/ontology/OntologyFactory.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/ontology/OntologyFactory.java (working copy) @@ -58,7 +58,9 @@ Extension extension = findExtension(extensionName); if (extension != null) { LOG.info("Using ontology extension: " + extensionName); - return (Ontology) extension.getExtensionInstance(); + Ontology o = (Ontology)extension.getExtensionInstance(); + o.setConf(conf); + return o; } LOG.warning("Ontology extension not found: '" + extensionName + "', trying the default"); @@ -69,7 +71,9 @@ if (extensions.length > 0) { LOG.info("Using the first ontology extension found: " + extensions[0].getId()); - return (Ontology) extensions[0].getExtensionInstance(); + Ontology o = (Ontology) extensions[0].getExtensionInstance(); + o.setConf(conf); + return o; } else { return null; } Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/plugin/PluginRepository.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/plugin/PluginRepository.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/plugin/PluginRepository.java (working copy) @@ -88,6 +88,7 @@ installExtensions(fRegisteredPlugins); } catch (PluginRuntimeException e) { LOG.severe(e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); throw new RuntimeException(e.getMessage()); } displayStatus(); Index: /Users/mattmann/src/nutch/src/java/org/apache/nutch/util/NutchConfiguration.java =================================================================== --- /Users/mattmann/src/nutch/src/java/org/apache/nutch/util/NutchConfiguration.java (revision 413074) +++ /Users/mattmann/src/nutch/src/java/org/apache/nutch/util/NutchConfiguration.java (working copy) @@ -31,6 +31,9 @@ * resources. */ public class NutchConfiguration { + public final static String LOG_SEVERE_FIELD = "__LOG_SEVERE_FIELD__"; + public final static String LOG_SEVERE = "__ls__"; + private final static String KEY = NutchConfiguration.class.getName(); private NutchConfiguration() {} // singleton Index: /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java (working copy) @@ -28,6 +28,7 @@ import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.HtmlParseFilter; import org.apache.nutch.protocol.Content; +import org.apache.nutch.util.NutchConfiguration; // Hadoop imports import org.apache.hadoop.util.LogFormatter; @@ -52,6 +53,11 @@ /* A static Map of ISO-639 language codes */ private static Map LANGUAGES_MAP = new HashMap(); + + // this needs to be initialized in order for the setting for the LOG SEVERE field + // to not fail with an NPE + private static Configuration conf = NutchConfiguration.create(); + static { try { Properties p = new Properties(); @@ -68,10 +74,10 @@ } } catch (Exception e) { LOG.severe(e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } } - private Configuration conf; Index: /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java (working copy) @@ -153,6 +153,7 @@ is.close(); } catch (IOException e1) { LOG.severe(e1.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } } } @@ -171,6 +172,7 @@ suspect = new NGramProfile("suspect", minLength, maxLength); } catch (Exception e) { LOG.severe(e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } } Index: /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java (working copy) @@ -35,6 +35,8 @@ import java.util.logging.Logger; // Nutch imports +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.LogFormatter; // Lucene imports @@ -39,6 +41,7 @@ // Lucene imports import org.apache.lucene.analysis.Token; +import org.apache.nutch.util.NutchConfiguration; /** @@ -52,7 +55,7 @@ * @author Sami Siren * @author Jerome Charron - http://frutch.free.fr/ */ -public class NGramProfile { +public class NGramProfile implements Configurable{ public static final Logger LOG = LogFormatter .getLogger("org.apache.nutch.analysis.lang.NGramProfile"); @@ -102,6 +105,9 @@ /** A StringBuffer used during analysis */ private QuickStringBuffer word = new QuickStringBuffer(); + /* configuration needed to record log severe */ + private Configuration conf = NutchConfiguration.create(); + /** * Construct a new ngram profile @@ -118,6 +124,14 @@ this.name = name; } + //inherited javadoc + public Configuration getConf(){return conf;} + + //inherited javadoc + public void setConf(Configuration conf){ + this.conf = conf; + } + /** * @return Returns the name. */ @@ -318,6 +332,7 @@ } } catch (Exception e) { LOG.severe(e.toString()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } return sum; } Index: /Users/mattmann/src/nutch/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (working copy) @@ -115,7 +115,8 @@ private Logger logger = LOGGER; /** The nutch configuration */ - private Configuration conf = null; + /** updated to be protected because of inherited classes that may need to set LOG SEVERE */ + protected Configuration conf = null; /** Do we block by IP addresses or by hostnames? */ private boolean byIP = true; Index: /Users/mattmann/src/nutch/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (working copy) @@ -35,6 +35,7 @@ import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.protocol.ProtocolException; +import org.apache.nutch.util.NutchConfiguration; /** @@ -207,6 +208,7 @@ if (agents.size() == 0) { agents.add(agentName); LOG.severe("No agents listed in 'http.robots.agents' property!"); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } else if (!((String)agents.get(0)).equalsIgnoreCase(agentName)) { agents.add(0, agentName); LOG.severe("Agent we advertise (" + agentName @@ -211,6 +213,7 @@ agents.add(0, agentName); LOG.severe("Agent we advertise (" + agentName + ") not listed first in 'http.robots.agents' property!"); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } setRobotNames((String[]) agents.toArray(new String[agents.size()])); } Index: /Users/mattmann/src/nutch/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.LogFormatter; import org.apache.nutch.net.*; +import org.apache.nutch.util.NutchConfiguration; /** @@ -132,6 +133,7 @@ Reader reader = conf.getConfResourceAsReader(file); if (reader == null) { LOG.severe("Can't find resource: " + file); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); } else { try { rules = readRulesFile(reader); @@ -137,6 +139,7 @@ rules = readRulesFile(reader); } catch (IOException e) { LOG.severe(e.getMessage()); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); //TODO mb@media-style.com: throw Exception? Because broken api. throw new RuntimeException(e.getMessage(), e); } Index: /Users/mattmann/src/nutch/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java =================================================================== --- /Users/mattmann/src/nutch/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java (revision 413074) +++ /Users/mattmann/src/nutch/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java (working copy) @@ -66,6 +66,9 @@ private static Map m_anonIDs = new HashMap(); private static int m_anonCount = 0; + + /* hadoop configuration needed to flag log severes */ + private Configuration conf = null; public OntologyImpl() { //only initialize all the static variables @@ -82,6 +85,14 @@ //ModelFactory.createOntologyModel(); } + //inherited javadoc + public void setConf(Configuration conf){ + this.conf = conf; + } + + //inherited javadoc + public Configuration getConf(){return conf;} + public static Ontology getInstance () { if (ontology == null) { //ontology = new org.apache.nutch.ontology.Ontology(); @@ -105,6 +116,7 @@ ((OntModel)m).read(url); } catch (Exception e) { LOG.severe("failed on attempting to read ontology "+url); + conf.set(NutchConfiguration.LOG_SEVERE_FIELD, NutchConfiguration.LOG_SEVERE); LOG.severe(e.getMessage()); StackTraceElement[] traces = e.getStackTrace(); for (int i=0; i