Index: src/java/org/apache/nutch/tools/MongodbParser.java =================================================================== --- src/java/org/apache/nutch/tools/MongodbParser.java (revision 0) +++ src/java/org/apache/nutch/tools/MongodbParser.java (revision 0) @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.tools; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.Set; +import java.util.regex.Pattern; + +// mongodb +import com.mongodb.*; + +// Slf4j Logging imports +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility that converts mongodb collection record into a flat file of URLs to be injected. + */ +public class MongodbParser { + public static final Logger LOG = LoggerFactory.getLogger(MongodbParser.class); + + private static final String DEFAULT_MONGODB_COLLECTION = "seed"; + private static final int FIELD_SORT_ORDER_DEC = -1; + private static final int FIELD_SORT_ORDER_ASC = 1; + + /** + * Iterate through all the items in this from the mongodb query file. + * Add each URL to the web db. + */ + public void parseMongodb(String mUri, String collection, String query, boolean isQueryRegex, String fields, String outputFieldNames,int limit, String sortby, int sortOrder) { + Mongo mongo = null; + try { + MongoURI mongoURI = new MongoURI(mUri); + + mongo = mongoURI.connect(); + DB db = mongo.getDB(mongoURI.getDatabase()); + DBCollection col = db.getCollection(collection); + + // Get the list of sources + BasicDBObject search = new BasicDBObject(); + DBCursor cursor = null; + // Get query + if (query != null) { + int fsemi = query.indexOf(":"); + if (fsemi != -1) { + String key = query.substring(0, fsemi); + String value = query.substring(fsemi + 1, query.length()); + if (isQueryRegex) { + Pattern match = Pattern.compile(value, Pattern.CASE_INSENSITIVE); + search.put(key, match); + } else + search.put(key, value); + } + } + + // Get fields name + BasicDBObject retFields = new BasicDBObject(); + if (fields != null) { + String[] values = fields.split(","); + for (String value : values) + retFields.append(value, 1); + } + + if (limit != Integer.MAX_VALUE) { + if (sortby != null) { + BasicDBObject sortField = new BasicDBObject(); + sortField.append(sortby, sortOrder); + cursor = col.find(search, retFields).sort(sortField).limit(limit); + } else + cursor = col.find(search, retFields).limit(limit); + } else { + if (sortby != null) { + BasicDBObject sortField = new BasicDBObject(); + sortField.append(sortby, sortOrder); + cursor = col.find(search, retFields).sort(sortField); + } else + cursor = col.find(search, retFields); + } + + String[] outputfn = outputFieldNames.split(","); + while (cursor.hasNext()) { + cursor.next(); + DBObject object = cursor.curr(); + Set keys = object.keySet(); + + StringBuffer sb = new StringBuffer(); + int i = 0; + for (String key : keys) { + // skip object id field + if (key.equals("_id")) + continue; + + // concate the url record + if(outputfn.length > i && outputfn[i].length() > 0) + sb.append(outputfn[i] + "="); + if(object.get(key) instanceof String) + sb.append(object.get(key).toString()); + else if(object.get(key) instanceof Integer) + sb.append(String.valueOf((Integer)object.get(key))); + else if(object.get(key) instanceof Float) + sb.append(String.valueOf((Float)object.get(key))); + else if(object.get(key) instanceof Double) + sb.append(String.valueOf((Double)object.get(key))); + else + { + LOG.warn("The field is convert to String. " + key + " with type " + object.get(key).getClass().getName()); + sb.append(object.get(key).toString()); + } + sb.append("\t"); + i++; + } + System.out.println(sb.toString()); + } + } catch (MongoException e) { + if (LOG.isErrorEnabled()) { + LOG.error(e.toString()); + } + System.exit(0); + } catch (UnknownHostException e) { + if (LOG.isErrorEnabled()) { + LOG.error(e.toString()); + } + System.exit(0); + } catch (IOException e) { + if (LOG.isErrorEnabled()) { + LOG.error(e.toString()); + } + System.exit(0); + } finally { + mongo.close(); + mongo = null; + } + + } + + /** + * Command-line access. User may add URLs from a mongodb database collection + * for a specified query. + */ + public static void main(String argv[]) throws Exception { + if (argv.length < 1) { + System.err.println("\n" + + "Usage: MongodbParser [-collection ] [-query ] [-queryRegex] [-limit ] [-fields ] [-outputFieldNames <,nutch.score,nutch.fetchInterval>] [-sortBy ] [-sortOrder ]"); + return; + } + + // + // Parse the command line, figure out what kind of + // URL file we need to load + // + String mongodbUri = argv[0]; // mongodb URI + String collection = MongodbParser.DEFAULT_MONGODB_COLLECTION; // collection name + String query = null; // query value + int limit = Integer.MAX_VALUE; // return number + String queryFields = null; // sort by field + String sortby = null; // sort order + String outputFieldNames = null; // output field name + int sortOrder = MongodbParser.FIELD_SORT_ORDER_DEC; + boolean regex = false; // is query by regex + + try { + for (int i = 1; i < argv.length; i++) { + if ("-collection".equals(argv[i])) { + collection = argv[i + 1]; + i++; + } else if ("-query".equals(argv[i])) { + query = argv[i + 1]; + i++; + } else if ("-queryRegex".equals(argv[i])) { + regex = true; + } else if ("-limit".equals(argv[i])) { + try { + limit = Integer.parseInt(argv[i + 1]); + } catch (NumberFormatException e) { + } + i++; + } else if ("-fields".equals(argv[i])) { + queryFields = argv[i + 1]; + i++; + }else if ("-outputFieldNames".equals(argv[i])) + { + outputFieldNames = argv[i+1]; + i++; + }else if ("-sortBy".equals(argv[i])) { + sortby = argv[i + 1]; + i++; + } else if ("-sortOrder".equals(argv[i])) { + String order = argv[i + 1]; + if (order.equals("dec")) + sortOrder = MongodbParser.FIELD_SORT_ORDER_DEC; + else if (order.equals("asc")) + sortOrder = MongodbParser.FIELD_SORT_ORDER_ASC; + i++; + } + } + MongodbParser parser = new MongodbParser(); + parser.parseMongodb(mongodbUri, collection, query, regex, queryFields, outputFieldNames, limit,sortby, sortOrder); + } catch (Exception e) { + LOG.error("MongodbParser: " + StringUtils.stringifyException(e)); + } + } +} \ No newline at end of file Index: pom.xml =================================================================== --- pom.xml (revision 1449609) +++ pom.xml (working copy) @@ -295,6 +295,12 @@ 6.1.22 true + + org.mongodb + mongo-java-driver + 2.9.3 + true + Index: ivy/ivy.xml =================================================================== --- ivy/ivy.xml (revision 1449609) +++ ivy/ivy.xml (working copy) @@ -30,6 +30,9 @@ + +