diff -Naur ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/ivy.xml src/plugin/indexer-elastic/ivy.xml
--- ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/ivy.xml 2017-03-29 04:06:50.000000000 +0000
+++ src/plugin/indexer-elastic/ivy.xml 2017-04-27 19:28:10.458777206 +0000
@@ -36,8 +36,10 @@
-
+
+
+
+
diff -Naur ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/plugin.xml src/plugin/indexer-elastic/plugin.xml
--- ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/plugin.xml 2017-03-29 04:06:50.000000000 +0000
+++ src/plugin/indexer-elastic/plugin.xml 2017-04-27 19:28:10.490777159 +0000
@@ -22,38 +22,64 @@
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff -Naur ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
--- ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java 2017-03-29 04:06:50.000000000 +0000
+++ src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java 2017-04-27 19:28:10.490777159 +0000
@@ -17,8 +17,6 @@
package org.apache.nutch.indexwriter.elastic;
-import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
-
import java.lang.invoke.MethodHandles;
import java.io.BufferedReader;
import java.io.IOException;
@@ -32,6 +30,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.nutch.indexer.IndexWriter;
import org.apache.nutch.indexer.NutchDocument;
+
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BackoffPolicy;
@@ -46,6 +45,8 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.node.Node;
+import org.elasticsearch.transport.client.*;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,7 +107,7 @@
String[] hosts = conf.getStrings(ElasticConstants.HOSTS);
int port = conf.getInt(ElasticConstants.PORT, DEFAULT_PORT);
- Settings.Builder settingsBuilder = Settings.settingsBuilder();
+ Settings.Builder settingsBuilder = Settings.builder();
BufferedReader reader = new BufferedReader(
conf.getConfResourceAsReader("elasticsearch.conf"));
@@ -133,12 +134,13 @@
// Prefer TransportClient
if (hosts != null && port > 1) {
- TransportClient transportClient = TransportClient.builder().settings(settings).build();
+ TransportClient transportClient = new PreBuiltTransportClient(settings);
+
for (String host: hosts)
transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
client = transportClient;
} else if (clusterName != null) {
- node = nodeBuilder().settings(settings).client(true).node();
+ node = new Node(settings);
client = node.client();
}
diff -Naur ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java
--- ../tmp/apache-nutch-1.13/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java 2017-03-29 04:06:50.000000000 +0000
+++ src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java 2017-05-01 14:25:07.818106789 +0000
@@ -28,6 +28,7 @@
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.DocWriteRequest.OpType;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -35,13 +36,14 @@
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.client.support.Headers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.threadpool.ThreadPool;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.util.UUID;
public class TestElasticIndexWriter {
@@ -63,10 +65,9 @@
Settings settings = Settings.builder().build();
ThreadPool threadPool = new ThreadPool(settings);
- Headers headers = new Headers(settings);
// customize the ES client to simulate responses from an ES cluster
- client = new AbstractClient(settings, threadPool, headers) {
+ client = new AbstractClient(settings, threadPool) {
@Override
public void close() { }
@@ -84,14 +85,14 @@
}
// respond with a failure
- BulkItemResponse failed = new BulkItemResponse(0, "index",
+ BulkItemResponse failed = new BulkItemResponse(0, OpType.INDEX,
new BulkItemResponse.Failure("nutch", "index", "failure0",
new EsRejectedExecutionException("saturated")));
response = new BulkResponse(new BulkItemResponse[]{failed}, 0);
} else {
// respond successfully
- BulkItemResponse success = new BulkItemResponse(0, "index",
- new IndexResponse("nutch", "index", "index0", 0, true));
+ BulkItemResponse success = new BulkItemResponse(0, OpType.INDEX,
+ new IndexResponse(new ShardId("nutch", UUID.randomUUID().toString(), 0), "index", "index0", 0, true));
response = new BulkResponse(new BulkItemResponse[]{success}, 0);
}