diff --git build.xml build.xml index c29ba7c..6251973 100644 --- build.xml +++ build.xml @@ -154,13 +154,16 @@ + + + @@ -576,6 +579,7 @@ + @@ -940,6 +944,7 @@ + @@ -955,6 +960,8 @@ + + + + + index.parse.md + description,keywords + + Comma-separated list of keys to be taken from the parse metadata to generate fields. + Can be used e.g. for 'description' or 'keywords' provided that these values are generated + by a parser (see parse-metatags plugin) + + + + + index.content.md + + + Comma-separated list of keys to be taken from the content metadata to generate fields. + + + + + index.db.md + + + Comma-separated list of keys to be taken from the crawldb metadata to generate fields. + Can be used to index values propagated from the seeds with the plugin urlmeta + + + + + + metatags.names + description;keywords + Names of the metatags to extract, separated by;. + Use '*' to extract all metatags. Prefixes the names with 'metatag.' + in the parse-metadata. For instance to index description and keywords, + you need to activate the plugin index-metadata and set the value of the + parameter 'index.parse.md' to 'metatag.description;metatag.keywords'. + + + diff --git src/plugin/build.xml src/plugin/build.xml index eaffa7b..a7f2a22 100755 --- src/plugin/build.xml +++ src/plugin/build.xml @@ -30,6 +30,7 @@ + @@ -47,6 +48,7 @@ + @@ -77,6 +79,7 @@ + @@ -112,6 +115,7 @@ + @@ -127,6 +131,7 @@ + diff --git src/plugin/index-metadata/build.xml src/plugin/index-metadata/build.xml new file mode 100644 index 0000000..ad96d11 --- /dev/null +++ src/plugin/index-metadata/build.xml @@ -0,0 +1,22 @@ + + + + + + + diff --git src/plugin/index-metadata/ivy.xml src/plugin/index-metadata/ivy.xml new file mode 100644 index 0000000..24d7606 --- /dev/null +++ src/plugin/index-metadata/ivy.xml @@ -0,0 +1,41 @@ + + + + + + + + + + Apache Nutch + + + + + + + + + + + + + + + + diff --git src/plugin/index-metadata/plugin.xml src/plugin/index-metadata/plugin.xml new file mode 100644 index 0000000..4d4c9a7 --- /dev/null +++ src/plugin/index-metadata/plugin.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + diff --git src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java new file mode 100644 index 0000000..0568d73 --- /dev/null +++ src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.indexer.metadata; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.avro.util.Utf8; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +// import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.indexer.IndexingException; +import org.apache.nutch.indexer.IndexingFilter; +import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.parse.Parse; +import org.apache.nutch.storage.WebPage; +import org.apache.nutch.storage.WebPage.Field; +import org.apache.nutch.util.Bytes; + +/** + * Indexer which can be configured to extract metadata from the crawldb, parse + * metadata or content metadata. You can specify the properties "index.db", + * "index.parse" or "index.content" who's values are comma-delimited + * key1, key2, key3. + */ + +public class MetadataIndexer implements IndexingFilter { + private Configuration conf; + private HashMap staticfields; + private static String[] dbFieldnames; + private static String[] parseFieldnames; + private static String[] contentFieldnames; + private static final String db_CONF_PROPERTY = "index.db.md"; + private static final String parse_CONF_PROPERTY = "index.parse.md"; + private static final String content_CONF_PROPERTY = "index.content.md"; + + public NutchDocument filter(NutchDocument doc, String url, WebPage page) + throws IndexingException { + + // just in case + if (doc == null) + return doc; + + // add the fields from crawldb + if (dbFieldnames != null) { + for (String metatag : dbFieldnames) { + ByteBuffer bmetadatas = page.getFromMetadata(new Utf8(metatag)); + String metadata = new String(bmetadatas.array()); + if (metadata != null) + doc.add("meta_" + metatag, metadata.toString()); + } + } + + // add the fields from parsemd + if (parseFieldnames != null) { + for (String metatag : parseFieldnames) { + ByteBuffer bvalues = page.getFromMetadata(new Utf8(metatag)); + if (bvalues != null) { + String value = new String(bvalues.array()); + String[] values = value.split("\t"); + for (String eachvalue : values) { + if (values != null) + doc.add("meta_" + metatag, eachvalue); + } + } + } + } + + // add the fields from contentmd + if (contentFieldnames != null) { + for (String metatag : contentFieldnames) { + // String[] value = parse.getData().getContentMeta().getValues(metatag); + ByteBuffer bvalues = page.getFromMetadata(new Utf8(metatag)); + String value = new String(bvalues.array()); + if (value != null) + doc.add("meta_" + metatag, value); + + } + } + + return doc; + } + + public void setConf(Configuration conf) { + this.conf = conf; + dbFieldnames = conf.getStrings(db_CONF_PROPERTY); + parseFieldnames = conf.getStrings(parse_CONF_PROPERTY); + contentFieldnames = conf.getStrings(content_CONF_PROPERTY); + + // TODO check conflict between field names e.g. could have same label + // from different sources + + } + + public Configuration getConf() { + return this.conf; + } + + @Override + public Collection getFields() { + // TODO Auto-generated method stub + return null; + } +} diff --git src/plugin/parse-metatags/README.txt src/plugin/parse-metatags/README.txt new file mode 100644 index 0000000..17c5def --- /dev/null +++ src/plugin/parse-metatags/README.txt @@ -0,0 +1,22 @@ +Parse-metatags plugin + +The parse-metatags plugin consists of a HTMLParserFilter which takes as +parameter a list of metatag names with '*' as default value. The values +are separated by ';'. +In order to extract the values of the metatags description and keywords, +you must specify in nutch-site.xml + + + metatags.names + description;keywords + + +Prefixes the names with 'metatag.' in the parse-metadata. For instance to +index description and keywords, you need to activate the plugin index-metadata +and set the value of the parameter 'index.parse.md' to 'metatag.description;metatag.keywords'. + +This code has been developed by DigitalPebble Ltd and offered to the community by ANT.com + + + + diff --git src/plugin/parse-metatags/build.xml src/plugin/parse-metatags/build.xml new file mode 100644 index 0000000..63ccb95 --- /dev/null +++ src/plugin/parse-metatags/build.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git src/plugin/parse-metatags/ivy.xml src/plugin/parse-metatags/ivy.xml new file mode 100644 index 0000000..1a86d68 --- /dev/null +++ src/plugin/parse-metatags/ivy.xml @@ -0,0 +1,41 @@ + + + + + + + + + + Apache Nutch + + + + + + + + + + + + + + + + diff --git src/plugin/parse-metatags/plugin.xml src/plugin/parse-metatags/plugin.xml new file mode 100644 index 0000000..6285e9c --- /dev/null +++ src/plugin/parse-metatags/plugin.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + diff --git src/plugin/parse-metatags/sample/testMetatags.html src/plugin/parse-metatags/sample/testMetatags.html new file mode 100644 index 0000000..f056fd4 --- /dev/null +++ src/plugin/parse-metatags/sample/testMetatags.html @@ -0,0 +1,11 @@ + + + + + + + + +text of the document + + diff --git src/plugin/parse-metatags/sample/testMultivalueMetatags.html src/plugin/parse-metatags/sample/testMultivalueMetatags.html new file mode 100644 index 0000000..63ad158 --- /dev/null +++ src/plugin/parse-metatags/sample/testMultivalueMetatags.html @@ -0,0 +1,13 @@ + + + + + +meta name="keywords" lang="en" content="web crawler" /> + + + + +A test for multi-valued metatags. + + \ No newline at end of file diff --git src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java new file mode 100644 index 0000000..3421832 --- /dev/null +++ src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.parse; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Properties; +import java.util.Set; + +import org.apache.avro.util.Utf8; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.metadata.Metadata; +import org.apache.nutch.storage.WebPage; +import org.apache.nutch.storage.WebPage.Field; +import org.apache.nutch.util.Bytes; +import org.w3c.dom.DocumentFragment; + +/** + * Parse HTML meta tags (keywords, description) and store them in the parse + * metadata so that they can be indexed with the index-metadata plugin with the + * prefix 'metatag.' + ***/ + +public class MetaTagsParser implements ParseFilter { + + private static final Log LOG = LogFactory.getLog(MetaTagsParser.class + .getName()); + + private Configuration conf; + + private Set metatagset = new HashSet(); + + public void setConf(Configuration conf) { + this.conf = conf; + // specify whether we want a specific subset of metadata + // by default take everything we can find + String metatags = conf.get("metatags.names", "*"); + String[] values = metatags.split(";"); + for (String val : values) + metatagset.add(val.toLowerCase()); + } + + public Configuration getConf() { + return this.conf; + } + + public Parse filter(String url, WebPage page, Parse parse, + HTMLMetaTags metaTags, DocumentFragment doc) { + + Metadata metadata = new Metadata(); + StringBuilder sb; + + // check in the metadata first : the tika-parser + // might have stored the values there already + for (String mdName : metadata.names()) { + String value = metadata.get(mdName); + // check whether the name is in the list of what we want or if + // specified * + if (metatagset.contains("*") || metatagset.contains(mdName.toLowerCase())) { + LOG.debug("Found meta tag: '" + mdName + "', with value: '" + value + + "'"); + metadata.add("metatag." + mdName.toLowerCase(), value); + } + } + + Properties generalMetaTags = metaTags.getGeneralTags(); + for (Iterator it = generalMetaTags.keySet().iterator(); it + .hasNext();) { + sb = new StringBuilder(); + String name = (String) it.next(); + String[] values = new String[] { (String) generalMetaTags.get(name) }; + // The multivalues of a metadata field are saved with a separator '\t' in + // the storage + // unless there is only one entry, where no \t is appended. + for (String value : values) { + if (values.length > 1) { + sb.append(value + "\t"); + } else { + sb.append(value); + } + } + page.putToMetadata(new Utf8(name), + ByteBuffer.wrap(Bytes.toBytes(sb.toString()))); + + // check whether the name is in the list of what we want or if + // specified * + if (metatagset.contains("*") || metatagset.contains(name.toLowerCase())) { + for (int i = 0; i < values.length; i++) { + // Add the recently parsed value of multiValued array to metadata + LOG.warn("Found meta tag : " + name + "\t" + values[i]); + metadata.add("metatag." + name.toLowerCase(), values[i]); + } + } + } + + Properties httpequiv = metaTags.getHttpEquivTags(); + for (Enumeration tagNames = httpequiv.propertyNames(); tagNames + .hasMoreElements();) { + String name = (String) tagNames.nextElement(); + String value = httpequiv.getProperty(name); + // check whether the name is in the list of what we want or if + // specified * + if (metatagset.contains("*") || metatagset.contains(name.toLowerCase())) { + LOG.debug("Found meta tag : " + name + "\t" + value); + metadata.add("metatag." + name.toLowerCase(), value); + } + } + + return parse; + } + + @Override + public Collection getFields() { + // TODO Auto-generated method stub + return null; + } + +} diff --git src/plugin/parse-metatags/src/test/org/apache/nutch/parse/html/TestMetatagsParser.java src/plugin/parse-metatags/src/test/org/apache/nutch/parse/html/TestMetatagsParser.java new file mode 100644 index 0000000..ca24b20 --- /dev/null +++ src/plugin/parse-metatags/src/test/org/apache/nutch/parse/html/TestMetatagsParser.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.parse.html; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.apache.avro.util.Utf8; +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.parse.ParseUtil; +import org.apache.nutch.storage.WebPage; +import org.apache.nutch.util.Bytes; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.nio.ByteBuffer; +import java.util.Map; + +public class TestMetatagsParser { + + private String fileSeparator = System.getProperty("file.separator"); + private String sampleDir = System.getProperty("test.data", "."); + private String sampleFile = "testMetatags.html"; + private String description = "This is a test of description"; + private String keywords = "This is a test of keywords"; + + public Map parseMeta(String fileName, Configuration conf) { + try { + String urlString = "file:" + sampleDir + fileSeparator + fileName; + + File file = new File(sampleDir + fileSeparator + fileName); + byte[] bytes = new byte[(int) file.length()]; + DataInputStream in = new DataInputStream(new FileInputStream(file)); + in.readFully(bytes); + in.close(); + + WebPage page = new WebPage(); + page.setBaseUrl(new Utf8(urlString)); + page.setContent(ByteBuffer.wrap(bytes)); + page.setContentType(new Utf8("text/html")); + + ParseUtil parser = new ParseUtil(NutchConfiguration.create()); + parser.parse(urlString, page); + + return page.getMetadata(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.toString()); + return null; + } + } + + @Test + public void testIt() { + Configuration conf = NutchConfiguration.create(); + + // check that we get the same values + Map meta = parseMeta(sampleFile, conf); + + assertEquals(description, getMeta(meta, "description")); + assertEquals(keywords, getMeta(meta, "keywords")); + } + + private String getMeta(Map meta, String name) { + ByteBuffer raw = meta.get(new Utf8(name)); + return Bytes.toString(raw); + } + +}