diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index 6ddf964d..92aaa389 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -1645,6 +1645,15 @@ visit https://wiki.apache.org/nutch/SimilarityScoringFilter-->
+
+ index.metadata.separator
+
+
+ Separator to use if you want to index multiple values for a given field. Leave empty to
+ treat each value as a single value.
+
+
+
index.geoip.usage
diff --git a/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java b/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
index edb8b152..74d9eb1f 100644
--- a/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
+++ b/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
@@ -17,9 +17,12 @@
package org.apache.nutch.indexer.metadata;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -35,17 +38,21 @@ import org.apache.nutch.parse.Parse;
* Indexer which can be configured to extract metadata from the crawldb, parse
* metadata or content metadata. You can specify the properties "index.db.md",
* "index.parse.md" or "index.content.md" who's values are comma-delimited
- * Example : key1,key2,key3.
+ * key1,key2,key3.
*/
public class MetadataIndexer implements IndexingFilter {
private Configuration conf;
private String[] dbFieldnames;
private Map parseFieldnames;
private String[] contentFieldnames;
+ private String separator;
+ private Set mvFields;
private static final String db_CONF_PROPERTY = "index.db.md";
private static final String parse_CONF_PROPERTY = "index.parse.md";
private static final String content_CONF_PROPERTY = "index.content.md";
-
+ private static final String separator_CONF_PROPERTY = "index.metadata.separator";
+ private static final String mvfields_CONF_PROPERTY = "index.metadata.multivalued.fields";
+
public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
CrawlDatum datum, Inlinks inlinks) throws IndexingException {
@@ -58,7 +65,7 @@ public class MetadataIndexer implements IndexingFilter {
for (String metatag : dbFieldnames) {
Writable metadata = datum.getMetaData().get(new Text(metatag));
if (metadata != null)
- doc.add(metatag, metadata.toString());
+ add(doc, metatag, metadata.toString());
}
}
@@ -67,7 +74,7 @@ public class MetadataIndexer implements IndexingFilter {
for (String metatag : parseFieldnames.keySet()) {
for (String value : parse.getData().getParseMeta().getValues(metatag)) {
if (value != null)
- doc.add(parseFieldnames.get(metatag), value);
+ add(doc, parseFieldnames.get(metatag), value);
}
}
}
@@ -77,13 +84,27 @@ public class MetadataIndexer implements IndexingFilter {
for (String metatag : contentFieldnames) {
for (String value : parse.getData().getContentMeta().getValues(metatag)) {
if (value != null)
- doc.add(metatag, value);
+ add(doc, metatag, value);
}
}
}
return doc;
}
+
+ protected void add(NutchDocument doc, String key, String value) {
+ if (separator == null || value.indexOf(separator) == -1 || !mvFields.contains(key)) {
+ doc.add(key, value);
+ } else {
+ String[] parts = value.split(separator);
+ for (String part : parts) {
+ part = part.trim();
+ if (part.length() != 0) {
+ doc.add(key, part);
+ }
+ }
+ }
+ }
public void setConf(Configuration conf) {
this.conf = conf;
@@ -93,7 +114,9 @@ public class MetadataIndexer implements IndexingFilter {
parseFieldnames.put(metatag.toLowerCase(Locale.ROOT), metatag);
}
contentFieldnames = conf.getStrings(content_CONF_PROPERTY);
-
+
+ separator = conf.get(separator_CONF_PROPERTY, null);
+ mvFields = new HashSet(Arrays.asList(conf.getStrings(mvfields_CONF_PROPERTY, new String[0])));
// TODO check conflict between field names e.g. could have same label
// from different sources