diff --git src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
index cc712e0..bfbed11 100644
--- src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
+++ src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
@@ -16,6 +16,8 @@
*/
package org.apache.nutch.indexer.more;
+import java.io.File;
+import java.nio.ByteBuffer;
import java.text.ParseException;
import java.util.Collection;
import java.util.Date;
@@ -31,6 +33,7 @@ import org.apache.nutch.metadata.HttpHeaders;
import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
+import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.MimeUtil;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.MatchResult;
@@ -42,14 +45,19 @@ import org.apache.solr.common.util.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.maxmind.geoip.Location;
+import com.maxmind.geoip.LookupService;
+
/**
* Add (or reset) a few metaData properties as respective fields (if they are
* available), so that they can be accurately used within the search index.
*
- * 'lastModifed' is indexed to support query by date, 'contentLength' obtains content length from the HTTP
- * header, 'type' field is indexed to support query by type and finally the 'title' field is an attempt
- * to reset the title if a content-disposition hint exists. The logic is that such a presence is indicative
- * that the content provider wants the filename therein to be used as the title.
+ * 'lastModifed' is indexed to support query by date, 'contentLength' obtains
+ * content length from the HTTP header, 'type' field is indexed to support query
+ * by type and finally the 'title' field is an attempt to reset the title if a
+ * content-disposition hint exists. The logic is that such a presence is
+ * indicative that the content provider wants the filename therein to be used as
+ * the title.
*
* Still need to make content-length searchable!
*
@@ -57,7 +65,8 @@ import org.slf4j.LoggerFactory;
*/
public class MoreIndexingFilter implements IndexingFilter {
- public static final Logger LOG = LoggerFactory.getLogger(MoreIndexingFilter.class);
+ public static final Logger LOG = LoggerFactory
+ .getLogger(MoreIndexingFilter.class);
/** Get the MimeTypes resolver instance. */
private MimeUtil MIME;
@@ -76,10 +85,36 @@ public class MoreIndexingFilter implements IndexingFilter {
addTime(doc, page, url);
addLength(doc, page, url);
addType(doc, page, url);
+ addServerGeo(doc, page, url);
resetTitle(doc, page, url);
return doc;
}
+ private NutchDocument addServerGeo(NutchDocument doc, WebPage page, String url) {
+
+ if (conf.getBoolean("store.ip.address", false) == true) {
+ try {
+ ByteBuffer serverIpBytes = page.getFromMetadata(new Utf8("_ip_"));
+ if (serverIpBytes != null) {
+ String serverIp = Bytes.toString(serverIpBytes);
+ String pathDat = MoreIndexingFilter.class.getResource(".").getPath()
+ + File.separator + "GeoLiteCity.dat";
+ File file = new File(pathDat);
+ LookupService lookup = new LookupService(file,
+ LookupService.GEOIP_MEMORY_CACHE);
+ Location locationServices = lookup.getLocation(serverIp);
+ doc.add("location", String.valueOf(locationServices.latitude) + ","
+ + String.valueOf(locationServices.longitude));
+ }
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error(e.getMessage());
+ }
+ }
+ }
+ return doc;
+ }
+
// Add time related meta info. Add last-modified if present. Index date as
// last-modified, or, if that's not present, use fetch time.
private NutchDocument addTime(NutchDocument doc, WebPage page, String url) {
@@ -89,7 +124,8 @@ public class MoreIndexingFilter implements IndexingFilter {
// String lastModified = data.getMeta(Metadata.LAST_MODIFIED);
if (lastModified != null) { // try parse last-modified
time = getTime(lastModified.toString(), url); // use as time
- String formlastModified = DateUtil.getThreadLocalDateFormat().format(new Date(time));
+ String formlastModified = DateUtil.getThreadLocalDateFormat().format(
+ new Date(time));
// store as string
doc.add("lastModified", formlastModified);
}
@@ -98,7 +134,8 @@ public class MoreIndexingFilter implements IndexingFilter {
time = page.getModifiedTime(); // use Modified time
}
- String dateString = DateUtil.getThreadLocalDateFormat().format(new Date(time));
+ String dateString = DateUtil.getThreadLocalDateFormat().format(
+ new Date(time));
// un-stored, indexed and un-tokenized
doc.add("date", dateString);
@@ -113,17 +150,19 @@ public class MoreIndexingFilter implements IndexingFilter {
} catch (ParseException e) {
// try to parse it as date in alternative format
try {
- Date parsedDate = DateUtils.parseDate(date, new String[] {
- "EEE MMM dd HH:mm:ss yyyy", "EEE MMM dd HH:mm:ss yyyy zzz",
- "EEE MMM dd HH:mm:ss zzz yyyy", "EEE, dd MMM yyyy HH:mm:ss zzz",
- "EEE,dd MMM yyyy HH:mm:ss zzz", "EEE, dd MMM yyyy HH:mm:sszzz",
- "EEE, dd MMM yyyy HH:mm:ss", "EEE, dd-MMM-yy HH:mm:ss zzz",
- "yyyy/MM/dd HH:mm:ss.SSS zzz", "yyyy/MM/dd HH:mm:ss.SSS",
- "yyyy/MM/dd HH:mm:ss zzz", "yyyy/MM/dd", "yyyy.MM.dd HH:mm:ss",
- "yyyy-MM-dd HH:mm", "MMM dd yyyy HH:mm:ss. zzz",
- "MMM dd yyyy HH:mm:ss zzz", "dd.MM.yyyy HH:mm:ss zzz",
- "dd MM yyyy HH:mm:ss zzz", "dd.MM.yyyy; HH:mm:ss",
- "dd.MM.yyyy HH:mm:ss", "dd.MM.yyyy zzz", "yyyy-MM-dd'T'HH:mm:ss'Z'" });
+ Date parsedDate = DateUtils.parseDate(date,
+ new String[] { "EEE MMM dd HH:mm:ss yyyy",
+ "EEE MMM dd HH:mm:ss yyyy zzz", "EEE MMM dd HH:mm:ss zzz yyyy",
+ "EEE, dd MMM yyyy HH:mm:ss zzz",
+ "EEE,dd MMM yyyy HH:mm:ss zzz", "EEE, dd MMM yyyy HH:mm:sszzz",
+ "EEE, dd MMM yyyy HH:mm:ss", "EEE, dd-MMM-yy HH:mm:ss zzz",
+ "yyyy/MM/dd HH:mm:ss.SSS zzz", "yyyy/MM/dd HH:mm:ss.SSS",
+ "yyyy/MM/dd HH:mm:ss zzz", "yyyy/MM/dd", "yyyy.MM.dd HH:mm:ss",
+ "yyyy-MM-dd HH:mm", "MMM dd yyyy HH:mm:ss. zzz",
+ "MMM dd yyyy HH:mm:ss zzz", "dd.MM.yyyy HH:mm:ss zzz",
+ "dd MM yyyy HH:mm:ss zzz", "dd.MM.yyyy; HH:mm:ss",
+ "dd.MM.yyyy HH:mm:ss", "dd.MM.yyyy zzz",
+ "yyyy-MM-dd'T'HH:mm:ss'Z'" });
time = parsedDate.getTime();
// if (LOG.isWarnEnabled()) {
// LOG.warn(url + ": parsed date: " + date +" to:"+time);
@@ -175,7 +214,7 @@ public class MoreIndexingFilter implements IndexingFilter {
String mimeType = null;
Utf8 contentType = page.getContentType();
if (contentType == null)
- contentType = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_TYPE));
+ contentType = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_TYPE));
if (contentType == null) {
// Note by Jerome Charron on 20050415:
// Content Type not solved by a previous plugin
@@ -204,7 +243,7 @@ public class MoreIndexingFilter implements IndexingFilter {
if (conf.getBoolean("moreIndexingFilter.indexMimeTypeParts", true)) {
String[] parts = getParts(mimeType);
- for(String part: parts) {
+ for (String part : parts) {
doc.add("type", part);
}
}
diff --git src/plugin/index-more/build.xml src/plugin/index-more/build.xml
index dec1e12..826218c 100644
--- src/plugin/index-more/build.xml
+++ src/plugin/index-more/build.xml
@@ -17,6 +17,20 @@
-->
-
+
+
+
+ Copying dat file
+
+
+
+
+
+
+
+
+
+
+
diff --git src/plugin/index-more/ivy.xml src/plugin/index-more/ivy.xml
index 1a86d68..472a46a 100644
--- src/plugin/index-more/ivy.xml
+++ src/plugin/index-more/ivy.xml
@@ -1,41 +1,37 @@
-
+
-
-
-
-
- Apache Nutch
-
-
+
+
+
+
+ Apache Nutch
+
+
+
+
+
+
-
-
-
+
+
+
+
-
-
-
-
+
+
+
-
-
-
diff --git src/plugin/index-more/plugin.xml src/plugin/index-more/plugin.xml
index 545dfba..836f0cc 100644
--- src/plugin/index-more/plugin.xml
+++ src/plugin/index-more/plugin.xml
@@ -26,6 +26,7 @@
+
diff --git conf/solrindex-mapping.xml conf/solrindex-mapping.xml
index c39889c..94d657f 100644
--- conf/solrindex-mapping.xml
+++ conf/solrindex-mapping.xml
@@ -1,43 +1,33 @@
-
+
-
+
+
+
+
+
+
+
+
- Any fields in NutchDocument that match a name defined
- in field/@source will be renamed to the corresponding
- field/@dest.
- Additionally, if a field name (before mapping) matches
- a copyField/@source then its values will be copied to
- the corresponding copyField/@dest.
+
- uniqueKey has the same meaning as in Solr schema.xml
- and defaults to "id" if not defined.
- -->
-
-
-
-
-
-
-
-
id
diff --git conf/schema-solr4.xml conf/schema-solr4.xml
index f73cd4b..adc23f6 100644
--- conf/schema-solr4.xml
+++ conf/schema-solr4.xml
@@ -297,6 +297,8 @@
+
+
@@ -307,7 +309,8 @@
-
+
+
@@ -330,6 +333,8 @@
+
+