Index: conf/nutch-default.xml
===================================================================
--- conf/nutch-default.xml (revision 1022038)
+++ conf/nutch-default.xml (working copy)
@@ -1037,7 +1037,7 @@
storage.data.store.class
- org.gora.sql.store.SqlStore
+ org.apache.gora.sql.store.SqlStore
Default class for storing data
Index: ivy/ivy.xml
===================================================================
--- ivy/ivy.xml (revision 1022038)
+++ ivy/ivy.xml (working copy)
@@ -67,7 +67,7 @@
-
+
@@ -110,7 +110,7 @@
-
+
Index: ivy/ivysettings.xml
===================================================================
--- ivy/ivysettings.xml (revision 1022038)
+++ ivy/ivysettings.xml (working copy)
@@ -75,6 +75,6 @@
rather than look for them online.
-->
-
+
Index: src/test/nutch-site.xml
===================================================================
--- src/test/nutch-site.xml (revision 1022038)
+++ src/test/nutch-site.xml (working copy)
@@ -18,7 +18,7 @@
storage.data.store.class
- org.gora.sql.store.SqlStore
+ org.apache.gora.sql.store.SqlStore
Default class for storing data
Index: src/test/org/apache/nutch/storage/TestGoraStorage.java
===================================================================
--- src/test/org/apache/nutch/storage/TestGoraStorage.java (revision 1022038)
+++ src/test/org/apache/nutch/storage/TestGoraStorage.java (working copy)
@@ -12,8 +12,8 @@
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import org.gora.query.Result;
-import org.gora.store.DataStore;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
import junit.framework.TestCase;
Index: src/test/org/apache/nutch/crawl/TestInjector.java
===================================================================
--- src/test/org/apache/nutch/crawl/TestInjector.java (revision 1022038)
+++ src/test/org/apache/nutch/crawl/TestInjector.java (working copy)
@@ -32,12 +32,12 @@
import org.apache.nutch.util.AbstractNutchTest;
import org.apache.nutch.util.CrawlTestUtil;
import org.apache.nutch.util.TableUtil;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.sql.store.SqlStore;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
-import org.gora.util.ByteUtils;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.sql.store.SqlStore;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.util.ByteUtils;
import org.junit.Before;
/**
Index: src/test/org/apache/nutch/util/AbstractNutchTest.java
===================================================================
--- src/test/org/apache/nutch/util/AbstractNutchTest.java (revision 1022038)
+++ src/test/org/apache/nutch/util/AbstractNutchTest.java (working copy)
@@ -31,12 +31,12 @@
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.sql.store.SqlStore;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
-import org.gora.util.ByteUtils;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.sql.store.SqlStore;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.util.ByteUtils;
/**
* This class provides common routines for setup/teardown of an in-memory data
Index: src/test/org/apache/nutch/util/CrawlTestUtil.java
===================================================================
--- src/test/org/apache/nutch/util/CrawlTestUtil.java (revision 1022038)
+++ src/test/org/apache/nutch/util/CrawlTestUtil.java (working copy)
@@ -31,9 +31,9 @@
import org.apache.nutch.crawl.URLWebPage;
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.store.DataStore;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.handler.DefaultHandler;
Index: src/java/org/apache/nutch/fetcher/FetcherReducer.java
===================================================================
--- src/java/org/apache/nutch/fetcher/FetcherReducer.java (revision 1022038)
+++ src/java/org/apache/nutch/fetcher/FetcherReducer.java (working copy)
@@ -40,7 +40,7 @@
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.URLUtil;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
public class FetcherReducer
extends GoraReducer {
Index: src/java/org/apache/nutch/fetcher/FetchEntry.java
===================================================================
--- src/java/org/apache/nutch/fetcher/FetchEntry.java (revision 1022038)
+++ src/java/org/apache/nutch/fetcher/FetchEntry.java (working copy)
@@ -9,7 +9,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.nutch.storage.WebPage;
-import org.gora.util.IOUtils;
+import org.apache.gora.util.IOUtils;
public class FetchEntry extends Configured implements Writable {
Index: src/java/org/apache/nutch/fetcher/FetcherJob.java
===================================================================
--- src/java/org/apache/nutch/fetcher/FetcherJob.java (revision 1022038)
+++ src/java/org/apache/nutch/fetcher/FetcherJob.java (working copy)
@@ -25,7 +25,7 @@
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
/**
* Multi-threaded fetcher.
Index: src/java/org/apache/nutch/storage/ProtocolStatus.java
===================================================================
--- src/java/org/apache/nutch/storage/ProtocolStatus.java (revision 1022038)
+++ src/java/org/apache/nutch/storage/ProtocolStatus.java (working copy)
@@ -14,11 +14,11 @@
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificFixed;
-import org.gora.persistency.StateManager;
-import org.gora.persistency.impl.PersistentBase;
-import org.gora.persistency.impl.StateManagerImpl;
-import org.gora.persistency.StatefulHashMap;
-import org.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.ListGenericArray;
@SuppressWarnings("all")
public class ProtocolStatus extends PersistentBase {
@@ -63,6 +63,7 @@
}
@SuppressWarnings(value="unchecked")
public void put(int _field, Object _value) {
+ if(isFieldEqual(_field, _value)) return;
getStateManager().setDirty(this, _field);
switch (_field) {
case 0:code = (Integer)_value; break;
Index: src/java/org/apache/nutch/storage/StorageUtils.java
===================================================================
--- src/java/org/apache/nutch/storage/StorageUtils.java (revision 1022038)
+++ src/java/org/apache/nutch/storage/StorageUtils.java (working copy)
@@ -4,16 +4,16 @@
import java.util.Collection;
import java.util.Iterator;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraOutputFormat;
+import org.apache.gora.mapreduce.GoraReducer;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.query.Query;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Partitioner;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.mapreduce.GoraOutputFormat;
-import org.gora.mapreduce.GoraReducer;
-import org.gora.persistency.Persistent;
-import org.gora.query.Query;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
public class StorageUtils {
@@ -31,7 +31,7 @@
getDataStoreClass(Configuration conf) throws ClassNotFoundException {
return (Class extends DataStore>)
Class.forName(conf.get("storage.data.store.class",
- "org.gora.hbase.store.HBaseStore"));
+ "org.apache.gora.sql.store.SqlStore"));
}
public static void initMapperJob(Job job,
Index: src/java/org/apache/nutch/storage/WebTableCreator.java
===================================================================
--- src/java/org/apache/nutch/storage/WebTableCreator.java (revision 1022038)
+++ src/java/org/apache/nutch/storage/WebTableCreator.java (working copy)
@@ -1,7 +1,7 @@
package org.apache.nutch.storage;
import org.apache.nutch.util.NutchConfiguration;
-import org.gora.store.DataStore;
+import org.apache.gora.store.DataStore;
public class WebTableCreator {
public static void main(String[] args) throws Exception {
Index: src/java/org/apache/nutch/storage/WebPage.java
===================================================================
--- src/java/org/apache/nutch/storage/WebPage.java (revision 1022038)
+++ src/java/org/apache/nutch/storage/WebPage.java (working copy)
@@ -14,11 +14,11 @@
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificFixed;
-import org.gora.persistency.StateManager;
-import org.gora.persistency.impl.PersistentBase;
-import org.gora.persistency.impl.StateManagerImpl;
-import org.gora.persistency.StatefulHashMap;
-import org.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.ListGenericArray;
@SuppressWarnings("all")
public class WebPage extends PersistentBase {
@@ -124,6 +124,7 @@
}
@SuppressWarnings(value="unchecked")
public void put(int _field, Object _value) {
+ if(isFieldEqual(_field, _value)) return;
getStateManager().setDirty(this, _field);
switch (_field) {
case 0:baseUrl = (Utf8)_value; break;
Index: src/java/org/apache/nutch/storage/ParseStatus.java
===================================================================
--- src/java/org/apache/nutch/storage/ParseStatus.java (revision 1022038)
+++ src/java/org/apache/nutch/storage/ParseStatus.java (working copy)
@@ -14,11 +14,11 @@
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificFixed;
-import org.gora.persistency.StateManager;
-import org.gora.persistency.impl.PersistentBase;
-import org.gora.persistency.impl.StateManagerImpl;
-import org.gora.persistency.StatefulHashMap;
-import org.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.ListGenericArray;
@SuppressWarnings("all")
public class ParseStatus extends PersistentBase {
@@ -63,6 +63,7 @@
}
@SuppressWarnings(value="unchecked")
public void put(int _field, Object _value) {
+ if(isFieldEqual(_field, _value)) return;
getStateManager().setDirty(this, _field);
switch (_field) {
case 0:majorCode = (Integer)_value; break;
Index: src/java/org/apache/nutch/crawl/GeneratorReducer.java
===================================================================
--- src/java/org/apache/nutch/crawl/GeneratorReducer.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/GeneratorReducer.java (working copy)
@@ -12,7 +12,7 @@
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.URLUtil;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
/** Reduce class for generate
*
Index: src/java/org/apache/nutch/crawl/DbUpdateReducer.java
===================================================================
--- src/java/org/apache/nutch/crawl/DbUpdateReducer.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/DbUpdateReducer.java (working copy)
@@ -18,7 +18,7 @@
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.WebPageWritable;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
public class DbUpdateReducer
extends GoraReducer {
Index: src/java/org/apache/nutch/crawl/DbUpdaterJob.java
===================================================================
--- src/java/org/apache/nutch/crawl/DbUpdaterJob.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/DbUpdaterJob.java (working copy)
@@ -16,7 +16,7 @@
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
-import org.gora.mapreduce.StringComparator;
+import org.apache.gora.mapreduce.StringComparator;
public class DbUpdaterJob extends Configured
implements Tool {
Index: src/java/org/apache/nutch/crawl/WebTableReader.java
===================================================================
--- src/java/org/apache/nutch/crawl/WebTableReader.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/WebTableReader.java (working copy)
@@ -38,10 +38,10 @@
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.StringUtil;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.store.DataStore;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
/**
* Displays information about the entries of the webtable
Index: src/java/org/apache/nutch/crawl/InjectorJob.java
===================================================================
--- src/java/org/apache/nutch/crawl/InjectorJob.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/InjectorJob.java (working copy)
@@ -32,8 +32,8 @@
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.mapreduce.GoraOutputFormat;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraOutputFormat;
/** This class takes a flat file of URLs and adds them to the of pages to be
* crawled. Useful for bootstrapping the system.
Index: src/java/org/apache/nutch/crawl/GeneratorMapper.java
===================================================================
--- src/java/org/apache/nutch/crawl/GeneratorMapper.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/GeneratorMapper.java (working copy)
@@ -12,7 +12,7 @@
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
public class GeneratorMapper
extends GoraMapper {
Index: src/java/org/apache/nutch/crawl/DbUpdateMapper.java
===================================================================
--- src/java/org/apache/nutch/crawl/DbUpdateMapper.java (revision 1022038)
+++ src/java/org/apache/nutch/crawl/DbUpdateMapper.java (working copy)
@@ -15,7 +15,7 @@
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.WebPageWritable;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
public class DbUpdateMapper
extends GoraMapper {
Index: src/java/org/apache/nutch/parse/ParserJob.java
===================================================================
--- src/java/org/apache/nutch/parse/ParserJob.java (revision 1022038)
+++ src/java/org/apache/nutch/parse/ParserJob.java (working copy)
@@ -23,7 +23,7 @@
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.GoraMapper;
public class ParserJob extends GoraMapper
implements Tool {
Index: src/java/org/apache/nutch/util/IdentityPageReducer.java
===================================================================
--- src/java/org/apache/nutch/util/IdentityPageReducer.java (revision 1022038)
+++ src/java/org/apache/nutch/util/IdentityPageReducer.java (working copy)
@@ -3,7 +3,7 @@
import java.io.IOException;
import org.apache.nutch.storage.WebPage;
-import org.gora.mapreduce.GoraReducer;
+import org.apache.gora.mapreduce.GoraReducer;
public class IdentityPageReducer
extends GoraReducer {
Index: src/java/org/apache/nutch/util/WebPageWritable.java
===================================================================
--- src/java/org/apache/nutch/util/WebPageWritable.java (revision 1022038)
+++ src/java/org/apache/nutch/util/WebPageWritable.java (working copy)
@@ -8,7 +8,7 @@
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Writable;
import org.apache.nutch.storage.WebPage;
-import org.gora.util.IOUtils;
+import org.apache.gora.util.IOUtils;
public class WebPageWritable extends Configured
implements Writable {
Index: src/java/org/apache/nutch/util/domain/DomainStatistics.java
===================================================================
--- src/java/org/apache/nutch/util/domain/DomainStatistics.java (revision 1022038)
+++ src/java/org/apache/nutch/util/domain/DomainStatistics.java (working copy)
@@ -24,6 +24,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.query.Query;
+import org.apache.gora.store.DataStore;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -50,9 +53,6 @@
import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.TimingUtil;
import org.apache.nutch.util.URLUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.query.Query;
-import org.gora.store.DataStore;
/**
* Extracts some very basic statistics about domains from the crawldb
Index: src/java/org/apache/nutch/indexer/IndexerReducer.java
===================================================================
--- src/java/org/apache/nutch/indexer/IndexerReducer.java (revision 1022038)
+++ src/java/org/apache/nutch/indexer/IndexerReducer.java (working copy)
@@ -13,7 +13,7 @@
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.StringUtil;
import org.apache.nutch.util.TableUtil;
-import org.gora.store.DataStore;
+import org.apache.gora.store.DataStore;
public class IndexerReducer
extends Reducer {
Index: src/java/org/apache/nutch/indexer/IndexerJob.java
===================================================================
--- src/java/org/apache/nutch/indexer/IndexerJob.java (revision 1022038)
+++ src/java/org/apache/nutch/indexer/IndexerJob.java (working copy)
@@ -22,8 +22,8 @@
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TableUtil;
-import org.gora.mapreduce.GoraMapper;
-import org.gora.mapreduce.StringComparator;
+import org.apache.gora.mapreduce.GoraMapper;
+import org.apache.gora.mapreduce.StringComparator;
public abstract class IndexerJob
extends GoraMapper