Index: src/java/org/apache/nutch/indexer/solr/SolrConstants.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrConstants.java (revision 1501122) +++ src/java/org/apache/nutch/indexer/solr/SolrConstants.java (working copy) @@ -23,8 +23,6 @@ public static final String COMMIT_SIZE = SOLR_PREFIX + "commit.size"; - public static final String COMMIT_INDEX = SOLR_PREFIX + "commit.index"; - public static final String MAPPING_FILE = SOLR_PREFIX + "mapping.file"; public static final String USE_AUTH = SOLR_PREFIX + "auth"; @@ -33,16 +31,14 @@ public static final String PASSWORD = SOLR_PREFIX + "auth.password"; - public static final String PARAMS = SOLR_PREFIX + "params"; + public static final String COLLECTION = SOLR_PREFIX + "collection"; - public static final String ID_FIELD = "id"; + public static final String ZOOKEEPER_HOSTS = SOLR_PREFIX + "zookeeper.hosts"; - public static final String URL_FIELD = "url"; + @Deprecated + public static final String COMMIT_INDEX = SOLR_PREFIX + "commit.index"; - public static final String BOOST_FIELD = "boost"; - - public static final String TIMESTAMP_FIELD = "tstamp"; - - public static final String DIGEST_FIELD = "digest"; + @Deprecated + public static final String PARAMS = SOLR_PREFIX + "params"; } Index: src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java (revision 1501122) +++ src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java (working copy) @@ -187,7 +187,7 @@ /** Return each index as a split. */ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - SolrServer solr = SolrUtils.getCommonsHttpSolrServer(job); + SolrServer solr = SolrUtils.getSolrServer(job); final SolrQuery solrQuery = new SolrQuery(SOLR_GET_ALL_QUERY); solrQuery.setFields(SolrConstants.ID_FIELD); @@ -218,7 +218,7 @@ Reporter reporter) throws IOException { - SolrServer solr = SolrUtils.getCommonsHttpSolrServer(job); + SolrServer solr = SolrUtils.getSolrServer(job); SolrInputSplit solrSplit = (SolrInputSplit) split; final int numDocs = solrSplit.getNumDocs(); @@ -297,7 +297,7 @@ public void configure(JobConf job) { try { - solr = SolrUtils.getCommonsHttpSolrServer(job); + solr = SolrUtils.getSolrServer(job); noCommit = job.getBoolean("noCommit", false); } catch (MalformedURLException e) { throw new RuntimeException(e); Index: src/java/org/apache/nutch/indexer/solr/SolrUtils.java =================================================================== --- src/java/org/apache/nutch/indexer/solr/SolrUtils.java (revision 1501122) +++ src/java/org/apache/nutch/indexer/solr/SolrUtils.java (working copy) @@ -16,43 +16,149 @@ */ package org.apache.nutch.indexer.solr; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.UsernamePasswordCredentials; -import org.apache.commons.httpclient.params.HttpClientParams; +import java.util.ArrayList; +import java.util.List; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.params.HttpClientParams; +import org.apache.http.params.HttpParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.mapred.JobConf; -import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.impl.LBHttpSolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.CloudSolrServer; import java.net.MalformedURLException; +/** + * Handy Solr utility used for creating SolrJ clients and cleaning + * the input. + */ public class SolrUtils { public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class); - public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) throws MalformedURLException { - HttpClient client=new HttpClient(); + /** + * Returns a single SolrServer based on configuration. It will either create + * a single node HttpSolrServer or a CloudSolrServer. + * + * @param JobConf + * @return SolrServer + */ + public static SolrServer getSolrServer(JobConf job) throws MalformedURLException { + List ss = getSolrServers(job); + return ss.get(0); + } - // Check for username/password - if (job.getBoolean(SolrConstants.USE_AUTH, false)) { - String username = job.get(SolrConstants.USERNAME); + /** + * Returns a list of SolrServers based on configuration. It will either create + * a list of single node HttpSolrServers or a list of CloudSolrServers. Write, + * deletes and updates are sent to all SolrServers. + * + * @param JobConf + * @return SolrServer + */ + public static List getSolrServers(JobConf job) throws MalformedURLException { + String[] urls = job.getStrings(SolrConstants.SERVER_URL); + String[] zookeeper = job.getStrings(SolrConstants.ZOOKEEPER_HOSTS); - LOG.info("Authenticating as: " + username); + ArrayList ss = new ArrayList(); - AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME); + // Got URL's for a Zookeeper ensemble? We'll prefer that then! + if (zookeeper != null && zookeeper.length > 0) { + for (int i = 0; i < zookeeper.length; i++) { + ss.add(getCloudServer(zookeeper[i].replace('|', ','), job)); + } + } else { + for (int i = 0; i < urls.length; i++) { + ss.add(getHttpServer(urls[i], job)); + } + } - client.getState().setCredentials(scope, new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD))); + return ss; + } - HttpClientParams params = client.getParams(); - params.setAuthenticationPreemptive(true); + /** + * Shuts down the specified CloudSolrServer implementation. + * + * @param SolrServer + */ + public static void shutdown(SolrServer solrServer) { + if (solrServer instanceof CloudSolrServer) { + solrServer.shutdown(); + } + } + /** + * Get a HttpSolrServer for the given URL. + * + * @param String + * @param JobConf + * @return SolrServer + */ + private static SolrServer getHttpServer(String url, JobConf job) throws MalformedURLException { + LOG.info("SolrIndexer: opening HTTP connection to: " + url); + + return new HttpSolrServer(url, getHttpClient(job)); + } + + /** + * Get a CloudSolrServer for the given Zookeeper URL. + * + * @param String + * @param JobConf + * @return SolrServer + */ + private static SolrServer getCloudServer(String url, JobConf job) throws MalformedURLException { + LOG.info("SolrIndexer: opening Zookeeper connection to: " + url); + + CloudSolrServer ss = new CloudSolrServer(url); + ss.setDefaultCollection(job.get(SolrConstants.COLLECTION)); + ss.connect(); + return ss; + } + + /** + * Get a HttpClient with optional authentication. + * + * @param JobConf + * @return HttpClient + */ + private static org.apache.http.impl.client.AbstractHttpClient getHttpClient(JobConf job) { + // Get us a default HttpClient + DefaultHttpClient client = new DefaultHttpClient(); + + // Check for username/password and enable authentication if we have to + if (job.getBoolean(SolrConstants.USE_AUTH, false)) { + String username = job.get(SolrConstants.USERNAME); + LOG.info("Authenticating as: " + username); + + AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME); + + client.getCredentialsProvider().setCredentials(scope, + new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD))); + + HttpParams params = client.getParams(); + HttpClientParams.setAuthenticating(params, true); + client.setParams(params); } - - return new CommonsHttpSolrServer(job.get(SolrConstants.SERVER_URL), client); + + // Fine! + return client; } + /** + * Strips unicode non-char codepoints from the input string. Text content sent to Solr sometimes + * contains crap characters. Stripping them from the input makes it possible to transmit them + * without issues. + * + * @param String input + * @return Sting + */ public static String stripNonCharCodepoints(String input) { StringBuilder retval = new StringBuilder(); char ch; Index: src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java =================================================================== --- src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java (revision 1501122) +++ src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrConstants.java (working copy) @@ -31,6 +31,10 @@ public static final String PASSWORD = SOLR_PREFIX + "auth.password"; + public static final String COLLECTION = SOLR_PREFIX + "collection"; + + public static final String ZOOKEEPER_HOSTS = SOLR_PREFIX + "zookeeper.hosts"; + @Deprecated public static final String COMMIT_INDEX = SOLR_PREFIX + "commit.index"; Index: src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java =================================================================== --- src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java (revision 1501122) +++ src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java (working copy) @@ -16,45 +16,149 @@ */ package org.apache.nutch.indexwriter.solr; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.UsernamePasswordCredentials; -import org.apache.commons.httpclient.params.HttpClientParams; +import java.util.ArrayList; +import java.util.List; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.params.HttpClientParams; +import org.apache.http.params.HttpParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.mapred.JobConf; -import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.impl.LBHttpSolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.CloudSolrServer; import java.net.MalformedURLException; +/** + * Handy Solr utility used for creating SolrJ clients and cleaning + * the input. + */ public class SolrUtils { public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class); - public static CommonsHttpSolrServer getCommonsHttpSolrServer(JobConf job) throws MalformedURLException { - HttpClient client=new HttpClient(); + /** + * Returns a single SolrServer based on configuration. It will either create + * a single node HttpSolrServer or a CloudSolrServer. + * + * @param JobConf + * @return SolrServer + */ + public static SolrServer getSolrServer(JobConf job) throws MalformedURLException { + List ss = getSolrServers(job); + return ss.get(0); + } - // Check for username/password - if (job.getBoolean(SolrConstants.USE_AUTH, false)) { - String username = job.get(SolrConstants.USERNAME); + /** + * Returns a list of SolrServers based on configuration. It will either create + * a list of single node HttpSolrServers or a list of CloudSolrServers. Write, + * deletes and updates are sent to all SolrServers. + * + * @param JobConf + * @return SolrServer + */ + public static List getSolrServers(JobConf job) throws MalformedURLException { + String[] urls = job.getStrings(SolrConstants.SERVER_URL); + String[] zookeeper = job.getStrings(SolrConstants.ZOOKEEPER_HOSTS); - LOG.info("Authenticating as: " + username); + ArrayList ss = new ArrayList(); - AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME); + // Got URL's for a Zookeeper ensemble? We'll prefer that then! + if (zookeeper != null && zookeeper.length > 0) { + for (int i = 0; i < zookeeper.length; i++) { + ss.add(getCloudServer(zookeeper[i].replace('|', ','), job)); + } + } else { + for (int i = 0; i < urls.length; i++) { + ss.add(getHttpServer(urls[i], job)); + } + } - client.getState().setCredentials(scope, new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD))); + return ss; + } - HttpClientParams params = client.getParams(); - params.setAuthenticationPreemptive(true); + /** + * Shuts down the specified CloudSolrServer implementation. + * + * @param SolrServer + */ + public static void shutdown(SolrServer solrServer) { + if (solrServer instanceof CloudSolrServer) { + solrServer.shutdown(); + } + } + /** + * Get a HttpSolrServer for the given URL. + * + * @param String + * @param JobConf + * @return SolrServer + */ + private static SolrServer getHttpServer(String url, JobConf job) throws MalformedURLException { + LOG.info("SolrIndexer: opening HTTP connection to: " + url); + + return new HttpSolrServer(url, getHttpClient(job)); + } + + /** + * Get a CloudSolrServer for the given Zookeeper URL. + * + * @param String + * @param JobConf + * @return SolrServer + */ + private static SolrServer getCloudServer(String url, JobConf job) throws MalformedURLException { + LOG.info("SolrIndexer: opening Zookeeper connection to: " + url); + + CloudSolrServer ss = new CloudSolrServer(url); + ss.setDefaultCollection(job.get(SolrConstants.COLLECTION)); + ss.connect(); + return ss; + } + + /** + * Get a HttpClient with optional authentication. + * + * @param JobConf + * @return HttpClient + */ + private static org.apache.http.impl.client.AbstractHttpClient getHttpClient(JobConf job) { + // Get us a default HttpClient + DefaultHttpClient client = new DefaultHttpClient(); + + // Check for username/password and enable authentication if we have to + if (job.getBoolean(SolrConstants.USE_AUTH, false)) { + String username = job.get(SolrConstants.USERNAME); + LOG.info("Authenticating as: " + username); + + AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME); + + client.getCredentialsProvider().setCredentials(scope, + new UsernamePasswordCredentials(username, job.get(SolrConstants.PASSWORD))); + + HttpParams params = client.getParams(); + HttpClientParams.setAuthenticating(params, true); + client.setParams(params); } - - String serverURL = job.get(SolrConstants.SERVER_URL); - - return new CommonsHttpSolrServer(serverURL, client); + + // Fine! + return client; } + /** + * Strips unicode non-char codepoints from the input string. Text content sent to Solr sometimes + * contains crap characters. Stripping them from the input makes it possible to transmit them + * without issues. + * + * @param String input + * @return Sting + */ public static String stripNonCharCodepoints(String input) { StringBuilder retval = new StringBuilder(); char ch; Index: src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java =================================================================== --- src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java (revision 1501122) +++ src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java (working copy) @@ -55,7 +55,7 @@ private boolean delete = false; public void open(JobConf job, String name) throws IOException { - SolrServer server = SolrUtils.getCommonsHttpSolrServer(job); + SolrServer server = SolrUtils.getSolrServer(job); init(server, job); }