Index: conf/log4j.properties =================================================================== --- conf/log4j.properties (revision 1696319) +++ conf/log4j.properties (working copy) @@ -44,10 +44,8 @@ log4j.logger.org.apache.nutch.crawl.LinkDb=INFO,cmdstdout log4j.logger.org.apache.nutch.crawl.LinkDbMerger=INFO,cmdstdout log4j.logger.org.apache.nutch.indexer.IndexingJob=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.solr.SolrIndexer=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.solr.SolrWriter=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.solr.SolrDeleteDuplicates=INFO,cmdstdout -log4j.logger.org.apache.nutch.indexer.solr.SolrClean=INFO,cmdstdout +log4j.logger.org.apache.nutch.indexwriter.solr.SolrIndexWriter=INFO,cmdstdout +log4j.logger.org.apache.nutch.indexwriter.solr.SolrUtils-INFO,cmdstdout log4j.logger.org.apache.nutch.scoring.webgraph.WebGraph=INFO,cmdstdout log4j.logger.org.apache.nutch.scoring.webgraph.LinkRank=INFO,cmdstdout log4j.logger.org.apache.nutch.scoring.webgraph.Loops=INFO,cmdstdout Index: conf/nutch-default.xml =================================================================== --- conf/nutch-default.xml (revision 1696319) +++ conf/nutch-default.xml (working copy) @@ -1615,7 +1615,19 @@ + + solr.server.type + http + + Specifies the SolrServer implementation to use. This is a string value + of one of the following 'cloud', 'concurrent', 'http' or 'lb'. + The values represent CloudSolrServer, ConcurrentUpdateSolrServer, + HttpSolrServer or LBHttpSolrServer respectively. + + + + solr.server.url http://127.0.0.1:8983/solr/ @@ -1625,6 +1637,25 @@ + solr.zookeeper.url + + + Defines the Zookeeper URL which is an essential setting to be used + when using SolrCloud. This should be a fully qualified URL similar to + the property provided within 'solr.server.url' above. + + + + + solr.loadbalance.urls + + + A comma-seperated value representing the Solr servers to be used when + initiating LBHttpSolrServer as the SolrServer implementation. + + + + solr.mapping.file solrindex-mapping.xml Index: conf/schema-solr4.xml =================================================================== --- conf/schema-solr4.xml (revision 1696319) +++ conf/schema-solr4.xml (working copy) @@ -1,429 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - text - - - - - - - - - - - Index: conf/schema.xml =================================================================== --- conf/schema.xml (revision 1696319) +++ conf/schema.xml (working copy) @@ -1,72 +1,308 @@ + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + --> - + - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + - - + + - - + + - - - - - - + + + + + - - - - - + + - -+ + + - - + + + + + + - - id - content - + + - - - - - - - + + + + + + + id + text + + + + + + + + + + Index: ivy/ivy.xml =================================================================== --- ivy/ivy.xml (revision 1696319) +++ ivy/ivy.xml (working copy) @@ -34,25 +34,19 @@ - - + + - - - - - - + + + + + + + + @@ -69,8 +63,9 @@ - + + + @@ -79,40 +74,20 @@ - - - - - - - - - + + + + + - - - - - - - - - - - - + Index: src/java/org/apache/nutch/indexer/IndexingFilter.java =================================================================== --- src/java/org/apache/nutch/indexer/IndexingFilter.java (revision 1696319) +++ src/java/org/apache/nutch/indexer/IndexingFilter.java (working copy) @@ -19,7 +19,6 @@ // Hadoop imports import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; // Nutch imports Index: src/plugin/index-geoip/build-ivy.xml =================================================================== --- src/plugin/index-geoip/build-ivy.xml (revision 1693938) +++ src/plugin/index-geoip/build-ivy.xml (working copy) @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + Index: src/plugin/index-geoip/ivy.xml =================================================================== --- src/plugin/index-geoip/ivy.xml (revision 1696319) +++ src/plugin/index-geoip/ivy.xml (working copy) @@ -36,7 +36,11 @@ - + + + + + Index: src/plugin/index-geoip/plugin.xml =================================================================== --- src/plugin/index-geoip/plugin.xml (revision 1696319) +++ src/plugin/index-geoip/plugin.xml (working copy) @@ -25,15 +25,13 @@ - - - - - - - + + + + + Index: src/plugin/indexer-solr/build-ivy.xml =================================================================== --- src/plugin/indexer-solr/build-ivy.xml (revision 0) +++ src/plugin/indexer-solr/build-ivy.xml (working copy) @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: src/plugin/indexer-solr/ivy.xml =================================================================== --- src/plugin/indexer-solr/ivy.xml (revision 1696319) +++ src/plugin/indexer-solr/ivy.xml (working copy) @@ -36,8 +36,9 @@ - + + + Index: src/plugin/indexer-solr/plugin.xml =================================================================== --- src/plugin/indexer-solr/plugin.xml (revision 1696319) +++ src/plugin/indexer-solr/plugin.xml (working copy) @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - @@ -22,22 +22,17 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + @@ -45,9 +40,9 @@ - Index: src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java =================================================================== --- src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java (revision 1696319) +++ src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java (working copy) @@ -17,6 +17,7 @@ package org.apache.nutch.indexwriter.solr; public interface SolrConstants { + public static final String SOLR_PREFIX = "solr."; public static final String SERVER_URL = SOLR_PREFIX + "server.url"; @@ -30,7 +31,13 @@ public static final String USERNAME = SOLR_PREFIX + "auth.username"; public static final String PASSWORD = SOLR_PREFIX + "auth.password"; - + + public static final String SERVER_TYPE = SOLR_PREFIX + "server.type"; + + public static final String ZOOKEEPER_URL = SOLR_PREFIX + "zookeeper.url"; + + public static final String LOADBALANCE_URLS = SOLR_PREFIX + "loadbalance.urls"; + @Deprecated public static final String COMMIT_INDEX = SOLR_PREFIX + "commit.index"; Index: src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java =================================================================== --- src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java (revision 1696319) +++ src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java (working copy) @@ -55,7 +55,7 @@ private boolean delete = false; public void open(JobConf job, String name) throws IOException { - SolrServer server = SolrUtils.getCommonsHttpSolrServer(job); + SolrServer server = SolrUtils.getSolrServer(job); init(server, job); } @@ -183,7 +183,7 @@ config = conf; String serverURL = conf.get(SolrConstants.SERVER_URL); if (serverURL == null) { - String message = "Missing SOLR URL. Should be set via -D " + String message = "Missing Solr URL. Should be set via -D " + SolrConstants.SERVER_URL; message += "\n" + describe(); LOG.error(message); @@ -192,15 +192,20 @@ } public String describe() { - StringBuffer sb = new StringBuffer("SOLRIndexWriter\n"); + StringBuffer sb = new StringBuffer("SolrIndexWriter\n"); + sb.append("\t").append(SolrConstants.SERVER_TYPE) + .append(" : Type of SolrServer to communicate with (default 'http' however options include 'cloud', 'lb' and 'concurrent')\n"); sb.append("\t").append(SolrConstants.SERVER_URL) - .append(" : URL of the SOLR instance (mandatory)\n"); - sb.append("\t").append(SolrConstants.COMMIT_SIZE) - .append(" : buffer size when sending to SOLR (default 1000)\n"); + .append(" : URL of the Solr instance (mandatory)\n"); + sb.append("\t").append(SolrConstants.ZOOKEEPER_URL) + .append(" : URL of the Zookeeper URL (mandatory if 'cloud' value for solr.server.type)\n"); + sb.append("\t").append(SolrConstants.LOADBALANCE_URLS) + .append(" : Comma-separated string of Solr server strings to be used (madatory if 'lb' value for solr.server.type)\n"); sb.append("\t") .append(SolrConstants.MAPPING_FILE) - .append( - " : name of the mapping file for fields (default solrindex-mapping.xml)\n"); + .append(" : name of the mapping file for fields (default solrindex-mapping.xml)\n"); + sb.append("\t").append(SolrConstants.COMMIT_SIZE) + .append(" : buffer size when sending to Solr (default 1000)\n"); sb.append("\t").append(SolrConstants.USE_AUTH) .append(" : use authentication (default false)\n"); sb.append("\t").append(SolrConstants.USERNAME) @@ -209,5 +214,4 @@ .append(" : password for authentication\n"); return sb.toString(); } - } Index: src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java =================================================================== --- src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java (revision 1696319) +++ src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java (working copy) @@ -16,14 +16,20 @@ */ package org.apache.nutch.indexwriter.solr; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.UsernamePasswordCredentials; -import org.apache.commons.httpclient.params.HttpClientParams; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.mapred.JobConf; -import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; +import org.apache.solr.client.solrj.impl.CloudSolrServer; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.LBHttpSolrServer; +import org.apache.solr.client.solrj.SolrServer; import java.net.MalformedURLException; @@ -31,33 +37,62 @@ public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class); - public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) + private static SolrServer server; + + public static SolrServer getSolrServer(JobConf job) throws MalformedURLException { - HttpClient client = new HttpClient(); + boolean auth = job.getBoolean(SolrConstants.USE_AUTH, false); + + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); // Check for username/password - if (job.getBoolean(SolrConstants.USE_AUTH, false)) { + if (auth) { String username = job.get(SolrConstants.USERNAME); - LOG.info("Authenticating as: " + username); - AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME); + credentialsProvider.setCredentials(scope, + new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD))); + } + CloseableHttpClient client = + HttpClientBuilder.create().setDefaultCredentialsProvider(credentialsProvider).build(); - client.getState().setCredentials( - scope, - new UsernamePasswordCredentials(username, job - .get(SolrConstants.PASSWORD))); + String solrServer = job.get(SolrConstants.SERVER_TYPE, "http"); + String zkHost = job.get(SolrConstants.ZOOKEEPER_URL, null); + String solrServerUrl = job.get(SolrConstants.SERVER_URL); - HttpClientParams params = client.getParams(); - params.setAuthenticationPreemptive(true); - - client.setParams(params); + switch (solrServer) { + case "cloud": + server = new CloudSolrServer(zkHost); + LOG.debug("CloudSolrServer selected as indexing server."); + break; + case "concurrent": + server = new ConcurrentUpdateSolrServer(solrServerUrl, client, 1000, 10); + LOG.debug("ConcurrentUpdateSolrServer selected as indexing server."); + break; + case "http": + if (auth) { + server = new HttpSolrServer(solrServerUrl, client); + } else { + server = new HttpSolrServer(solrServerUrl); + } + LOG.debug("HttpSolrServer selected as indexing server."); + break; + case "lb": + String[] lbServerString = job.get(SolrConstants.LOADBALANCE_URLS).split(","); + server = new LBHttpSolrServer(client, lbServerString); + LOG.debug("LBHttpSolrServer selected as indexing server."); + break; + default: + if (auth) { + server = new HttpSolrServer(solrServerUrl, client); + } else { + server = new HttpSolrServer(solrServerUrl); + } + LOG.debug("HttpSolrServer selected as indexing server."); + break; } - - String serverURL = job.get(SolrConstants.SERVER_URL); - - return new CommonsHttpSolrServer(serverURL, client); + return server; } public static String stripNonCharCodepoints(String input) { @@ -82,4 +117,4 @@ return retval.toString(); } -} +} \ No newline at end of file Index: src/plugin/parse-tika/ivy.xml =================================================================== --- src/plugin/parse-tika/ivy.xml (revision 1696319) +++ src/plugin/parse-tika/ivy.xml (working copy) @@ -38,6 +38,8 @@ + + Index: src/plugin/parsefilter-naivebayes/build-ivy.xml =================================================================== --- src/plugin/parsefilter-naivebayes/build-ivy.xml (revision 1693468) +++ src/plugin/parsefilter-naivebayes/build-ivy.xml (working copy) @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + Index: src/plugin/parsefilter-naivebayes/ivy.xml =================================================================== --- src/plugin/parsefilter-naivebayes/ivy.xml (revision 1696319) +++ src/plugin/parsefilter-naivebayes/ivy.xml (working copy) @@ -36,6 +36,14 @@ + + + + + + + + Index: src/plugin/parsefilter-naivebayes/plugin.xml =================================================================== --- src/plugin/parsefilter-naivebayes/plugin.xml (revision 1696319) +++ src/plugin/parsefilter-naivebayes/plugin.xml (working copy) @@ -25,10 +25,22 @@ - - - - + + + + + + + + + + + + + + + + @@ -36,10 +48,9 @@ - + name="Nutch Parser Filter" point="org.apache.nutch.parse.HtmlParseFilter"> + Index: src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/NaiveBayesClassifier.java =================================================================== --- src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/NaiveBayesClassifier.java (revision 1696319) +++ src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/NaiveBayesClassifier.java (working copy) @@ -35,7 +35,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.util.Version; import org.apache.mahout.classifier.naivebayes.BayesUtils; import org.apache.mahout.classifier.naivebayes.NaiveBayesModel; import org.apache.mahout.classifier.naivebayes.StandardNaiveBayesClassifier; @@ -47,9 +46,6 @@ import org.apache.mahout.math.Vector.Element; import org.apache.mahout.vectorizer.SparseVectorsFromSequenceFiles; import org.apache.mahout.vectorizer.TFIDF; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.common.collect.ConcurrentHashMultiset; import com.google.common.collect.Multiset; @@ -56,8 +52,6 @@ public class NaiveBayesClassifier { private static NaiveBayesModel model = null; - private static final Logger LOG = LoggerFactory - .getLogger(NaiveBayesClassifier.class); public static Map readDictionnary(Configuration conf, Path dictionnaryPath) { @@ -114,7 +108,7 @@ public static String classify(String text, String modelPath, String labelIndexPath, String dictionaryPath, String documentFrequencyPath) - throws IOException { + throws IOException { Configuration configuration = new Configuration(); @@ -134,7 +128,7 @@ new Path(documentFrequencyPath)); // analyzer used to extract word from text - Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_43); + Analyzer analyzer = new StandardAnalyzer(); // int labelCount = labels.size(); int documentCount = documentFrequency.get(-1).intValue();