Fragment inner class */
- public void testFragment() {
- Fragment fragment = new Fragment("fragment text");
- assertEquals("fragment text", fragment.getText());
- assertEquals("fragment text", fragment.toString());
- assertFalse(fragment.isEllipsis());
- assertFalse(fragment.isHighlight());
- assertTrue(fragment.equals(new Fragment("fragment text")));
- assertFalse(fragment.equals(new Fragment("some text")));
- assertFalse(fragment.equals(new Ellipsis()));
- assertFalse(fragment.equals(new Highlight("fragment text")));
- }
-
- /** Test of Ellipsis inner class */
- public void testEllipsis() {
- Fragment fragment = new Ellipsis();
- assertEquals(" ... ", fragment.getText());
- assertEquals(" ... ", fragment.toString());
- assertTrue(fragment.isEllipsis());
- assertFalse(fragment.isHighlight());
- assertFalse(fragment.equals(new Fragment("fragment text")));
- assertTrue(fragment.equals(new Ellipsis()));
- assertFalse(fragment.equals(new Highlight("fragment text")));
- }
-
- /** Test of Highlight inner class */
- public void testHighlight() {
- Fragment fragment = new Highlight("highlight text");
- assertEquals("highlight text", fragment.getText());
- assertEquals("highlight text", fragment.toString());
- assertFalse(fragment.isEllipsis());
- assertTrue(fragment.isHighlight());
- assertFalse(fragment.equals(new Fragment("fragment text")));
- assertFalse(fragment.equals(new Ellipsis()));
- assertFalse(fragment.equals(new Highlight("fragment text")));
- assertTrue(fragment.equals(new Highlight("highlight text")));
- }
-
- /** Test of add / get methods */
- public void testAdd() {
- Fragment[] fragments = null;
- Summary summary = new Summary();
- summary.add(new Fragment("fragment1"));
- fragments = summary.getFragments();
- assertEquals(1, fragments.length);
- assertEquals("fragment1", fragments[0].toString());
- summary.add(new Fragment("fragment2"));
- fragments = summary.getFragments();
- assertEquals(2, fragments.length);
- assertEquals("fragment1", fragments[0].toString());
- assertEquals("fragment2", fragments[1].toString());
- summary.add(new Fragment("fragment3"));
- fragments = summary.getFragments();
- assertEquals(3, fragments.length);
- assertEquals("fragment1", fragments[0].toString());
- assertEquals("fragment2", fragments[1].toString());
- assertEquals("fragment3", fragments[2].toString());
- }
-
- /** Test of toString method. */
- public void testToString() {
- Summary summary = new Summary();
- assertEquals("", summary.toString());
- summary.add(new Fragment("fragment1"));
- assertEquals("fragment1", summary.toString());
- summary.add(new Ellipsis());
- assertEquals("fragment1 ... ", summary.toString());
- summary.add(new Highlight("highlight"));
- assertEquals("fragment1 ... highlight", summary.toString());
- summary.add(new Fragment("fragment2"));
- assertEquals("fragment1 ... highlightfragment2", summary.toString());
- }
-
- /** Test of toStrings. */
- public void testToStrings() {
- Summary[] summaries = { new Summary(), new Summary() };
- summaries[0].add(new Fragment("fragment1.1"));
- summaries[0].add(new Ellipsis());
- summaries[0].add(new Highlight("highlight1"));
- summaries[0].add(new Fragment("fragment1.2"));
- summaries[1].add(new Fragment("fragment2.1"));
- summaries[1].add(new Ellipsis());
- summaries[1].add(new Highlight("highlight2"));
- summaries[1].add(new Fragment("fragment2.2"));
- String[] strings = Summary.toStrings(summaries);
- assertEquals(2, strings.length);
- assertEquals("fragment1.1 ... highlight1fragment1.2", strings[0]);
- assertEquals("fragment2.1 ... highlight2fragment2.2", strings[1]);
- }
-
- /** Test of equals method. */
- public void testEquals() {
- Summary summary1 = new Summary();
- Summary summary2 = new Summary();
- assertFalse(summary1.equals(null));
- assertFalse(summary1.equals(""));
- assertTrue(summary1.equals(summary2));
- summary1.add(new Fragment("text fragment"));
- assertFalse(summary1.equals(summary2));
- summary2.add(new Fragment("text fragment"));
- assertTrue(summary1.equals(summary2));
- summary1.add(new Ellipsis());
- assertFalse(summary1.equals(summary2));
- summary2.add(new Ellipsis());
- assertTrue(summary1.equals(summary2));
- summary1.add(new Highlight("highlight"));
- assertFalse(summary1.equals(summary2));
- summary2.add(new Highlight("highlight"));
- assertTrue(summary1.equals(summary2));
- summary1.add(new Fragment("text fragment"));
- summary2.add(new Fragment("fragment text"));
- assertFalse(summary1.equals(summary2));
- }
-
- /** Test of writable implementation. */
- public void testWritable() throws Exception {
- Summary summary = new Summary();
- summary.add(new Fragment("fragment1.1"));
- summary.add(new Ellipsis());
- summary.add(new Highlight("highlight1"));
- summary.add(new Fragment("fragment1.2"));
- WritableTestUtils.testWritable(summary);
- }
-
-}
Index: src/test/org/apache/nutch/searcher/response/TestRequestUtils.java
===================================================================
--- src/test/org/apache/nutch/searcher/response/TestRequestUtils.java (revision 959954)
+++ src/test/org/apache/nutch/searcher/response/TestRequestUtils.java (working copy)
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nutch.searcher.response;
-
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.servlet.http.HttpServletRequest;
-
-import junit.framework.TestCase;
-
-public class TestRequestUtils extends TestCase {
-
- public TestRequestUtils(String name) {
- super(name);
- }
-
- /**
- * Test getBooleanParameter() - no default
- */
- public void testGetBooleanParameterNoDefault() {
- String param = "foo";
- Map parameters = new HashMap();
- HttpServletRequest request = createMockHttpServletRequest(parameters);
-
- assertFalse("No param", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "0");
- assertFalse("Foo=0", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "no");
- assertFalse("Foo=no", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "false");
- assertFalse("Foo=false", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "abcdef");
- assertFalse("Foo=abcdef", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "1");
- assertTrue("Foo=1", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "yes");
- assertTrue("Foo=yes", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "YES");
- assertTrue("Foo=YES", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "true");
- assertTrue("Foo=true", RequestUtils.getBooleanParameter(request, param));
-
- parameters.put(param, "TRUE");
- assertTrue("Foo=TRUE", RequestUtils.getBooleanParameter(request, param));
- }
-
- /**
- * Test getBooleanParameter() - with default
- */
- public void testGetBooleanParameterWithoDefault() {
- String param = "foo";
- Map parameters = new HashMap();
- HttpServletRequest request = createMockHttpServletRequest(parameters);
-
- assertTrue("No param - def true", RequestUtils.getBooleanParameter(request, param, true));
- assertFalse("No param - def false", RequestUtils.getBooleanParameter(request, param, false));
-
- parameters.put(param, "0");
- assertFalse("Foo=0", RequestUtils.getBooleanParameter(request, param, true));
-
- parameters.put(param, "no");
- assertFalse("Foo=no", RequestUtils.getBooleanParameter(request, param, true));
-
- parameters.put(param, "false");
- assertFalse("Foo=false", RequestUtils.getBooleanParameter(request, param, true));
-
- parameters.put(param, "abcdef");
- assertFalse("Foo=abcdef", RequestUtils.getBooleanParameter(request, param, true));
-
- parameters.put(param, "1");
- assertTrue("Foo=1", RequestUtils.getBooleanParameter(request, param, false));
-
- parameters.put(param, "yes");
- assertTrue("Foo=yes", RequestUtils.getBooleanParameter(request, param, false));
-
- parameters.put(param, "YES");
- assertTrue("Foo=YES", RequestUtils.getBooleanParameter(request, param, false));
-
- parameters.put(param, "true");
- assertTrue("Foo=true", RequestUtils.getBooleanParameter(request, param, false));
-
- parameters.put(param, "TRUE");
- assertTrue("Foo=TRUE", RequestUtils.getBooleanParameter(request, param, false));
- }
-
- /**
- * Create a mock HttpServletRequest.
- */
- private HttpServletRequest createMockHttpServletRequest(Map parameters) {
- MockHttpServletRequestInvocationHandler handler = new MockHttpServletRequestInvocationHandler();
- handler.setParameterMap(parameters);
- ClassLoader cl = getClass().getClassLoader();
- Class[] interfaces = new Class[] {HttpServletRequest.class};
- return (HttpServletRequest)Proxy.newProxyInstance(cl, interfaces, handler);
- }
-
- /**
- * InvocationHandler for mock HttpServletRequest proxy.
- */
- private static class MockHttpServletRequestInvocationHandler implements InvocationHandler {
- private Map parameters = new HashMap();
- public Object invoke(Object proxy, Method method, Object[] args)
- throws Throwable {
- if (method.getName().equals("getParameter")) {
- return parameters.get((String)args[0]);
- }
- return null;
- }
- public void setParameterMap(Map parameters) {
- this.parameters = parameters;
- }
- }
-}
\ No newline at end of file
Index: src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java
===================================================================
--- src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java (revision 959954)
+++ src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java (working copy)
@@ -1,204 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.nutch.indexer;
-
-import java.io.File;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.MD5Hash;
-import org.apache.lucene.document.DateTools;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.DateTools.Resolution;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriter.MaxFieldLength;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.nutch.analysis.NutchDocumentAnalyzer;
-import org.apache.nutch.util.NutchConfiguration;
-
-import junit.framework.TestCase;
-
-public class TestDeleteDuplicates extends TestCase {
- Configuration conf;
- FileSystem fs;
- Path root;
- Path index1;
- Path index2;
- Path index3;
- Path index4;
- Path index5;
-
- public void setUp() throws Exception {
- conf = NutchConfiguration.create();
- conf.set("fs.default.name", "file:///");
- fs = FileSystem.get(conf);
- root = new Path("build/test/dedup2-test-" + new Random().nextInt());
- // create test indexes
- index1 = createIndex("index1", true, 1.0f, 10L, false);
- index2 = createIndex("index2", false, 2.0f, 20L, true);
- index3 = createIndex("index3", true, 1.0f, 10L, true);
- index4 = createSingleDocIndex("index4", 1.0f, 10L);
- index5 = createSingleDocIndex("index5", 1.0f, 20L);
- }
-
- private Path createIndex(String name, boolean hashDup, float inc, long time, boolean incFirst) throws Exception {
- Path idx = new Path(root, name);
- Path sub = new Path(idx, "part-0000");
- Directory dir = FSDirectory.open(new File(sub.toString()));
- IndexWriter writer = new IndexWriter(dir, new NutchDocumentAnalyzer(conf), true,
- MaxFieldLength.UNLIMITED);
- Document doc = makeDoc(name,
- MD5Hash.digest("1").toString(),
- "http://www.example.com/1",
- 1.0f + (incFirst ? inc : 0.0f), time);
- writer.addDocument(doc);
- if (hashDup) {
- doc = makeDoc(name,
- MD5Hash.digest("1").toString(),
- "http://www.example.com/2",
- 1.0f + (!incFirst ? inc : 0.0f), time + 1);
- } else {
- doc = makeDoc(name,
- MD5Hash.digest("2").toString(),
- "http://www.example.com/1",
- 1.0f + (!incFirst ? inc : 0.0f), time + 1);
- }
- writer.addDocument(doc);
- writer.close();
- return idx;
- }
-
- private Path createSingleDocIndex(String name, float inc, long time) throws Exception {
- Path idx = new Path(root, name);
- Path sub = new Path(idx, "part-0000");
- Directory dir = FSDirectory.open(new File(sub.toString()));
- IndexWriter writer = new IndexWriter(dir, new NutchDocumentAnalyzer(conf), true,
- MaxFieldLength.UNLIMITED);
- Document doc = makeDoc(name,
- MD5Hash.digest("1").toString(),
- "http://www.example.com/1",
- 1.0f + inc, time + 1);
- writer.addDocument(doc);
- writer.close();
- return idx;
- }
-
- private Document makeDoc(String segment, String digest, String url, float boost, long time) {
- Document doc = new Document();
- doc.add(new Field("segment", segment, Field.Store.YES, Field.Index.NO));
- doc.add(new Field("digest", digest, Field.Store.YES, Field.Index.NO));
- doc.add(new Field("url", url, Field.Store.YES, Field.Index.ANALYZED));
- doc.setBoost(boost);
- doc.add(new Field("boost", "" + boost, Field.Store.YES, Field.Index.NO));
- doc.add(new Field("tstamp", DateTools.timeToString(time, Resolution.MILLISECOND), Field.Store.YES, Field.Index.NO));
- return doc;
- }
-
- public void tearDown() throws Exception {
- fs.delete(root, true);
- }
-
- private void hashDuplicatesHelper(Path index, String url) throws Exception {
- DeleteDuplicates dedup = new DeleteDuplicates(conf);
- dedup.dedup(new Path[]{index});
- FsDirectory dir = new FsDirectory(fs, new Path(index, "part-0000"), false, conf);
- IndexReader reader = IndexReader.open(dir);
- assertEquals("only one doc left", reader.numDocs(), 1);
- for (int i = 0; i < reader.maxDoc(); i++) {
- if (reader.isDeleted(i)) {
- System.out.println("-doc " + i + " deleted");
- continue;
- }
- Document doc = reader.document(i);
- // make sure we got the right one
- assertEquals("check url", url, doc.get("url"));
- System.out.println(doc);
- }
- reader.close();
- }
-
- public void testHashDuplicates() throws Exception {
- hashDuplicatesHelper(index1, "http://www.example.com/2");
- hashDuplicatesHelper(index3, "http://www.example.com/1");
- }
-
- public void testUrlDuplicates() throws Exception {
- DeleteDuplicates dedup = new DeleteDuplicates(conf);
- dedup.dedup(new Path[]{index2});
- FsDirectory dir = new FsDirectory(fs, new Path(index2, "part-0000"), false, conf);
- IndexReader reader = IndexReader.open(dir);
- assertEquals("only one doc left", reader.numDocs(), 1);
- MD5Hash hash = MD5Hash.digest("2");
- for (int i = 0; i < reader.maxDoc(); i++) {
- if (reader.isDeleted(i)) {
- System.out.println("-doc " + i + " deleted");
- continue;
- }
- Document doc = reader.document(i);
- // make sure we got the right one
- assertEquals("check hash", hash.toString(), doc.get("digest"));
- System.out.println(doc);
- }
- reader.close();
- }
-
- public void testMixedDuplicates() throws Exception {
- DeleteDuplicates dedup = new DeleteDuplicates(conf);
- dedup.dedup(new Path[]{index1, index2});
- FsDirectory dir = new FsDirectory(fs, new Path(index1, "part-0000"), false, conf);
- IndexReader reader = IndexReader.open(dir);
- assertEquals("only one doc left", reader.numDocs(), 1);
- for (int i = 0; i < reader.maxDoc(); i++) {
- if (reader.isDeleted(i)) {
- System.out.println("-doc " + i + " deleted");
- continue;
- }
- Document doc = reader.document(i);
- // make sure we got the right one
- assertEquals("check url", "http://www.example.com/2", doc.get("url"));
- System.out.println(doc);
- }
- reader.close();
- dir = new FsDirectory(fs, new Path(index2, "part-0000"), false, conf);
- reader = IndexReader.open(dir);
- assertEquals("only one doc left", reader.numDocs(), 1);
- MD5Hash hash = MD5Hash.digest("2");
- for (int i = 0; i < reader.maxDoc(); i++) {
- if (reader.isDeleted(i)) {
- System.out.println("-doc " + i + " deleted");
- continue;
- }
- Document doc = reader.document(i);
- // make sure we got the right one
- assertEquals("check hash", hash.toString(), doc.get("digest"));
- System.out.println(doc);
- }
- reader.close();
- }
-
- public void testRededuplicate() throws Exception {
- DeleteDuplicates dedup = new DeleteDuplicates(conf);
- dedup.dedup(new Path[]{index4, index5});
- dedup.dedup(new Path[]{index4, index5});
- }
-
-}
Index: src/test/org/apache/nutch/indexer/TestIndexSorter.java
===================================================================
--- src/test/org/apache/nutch/indexer/TestIndexSorter.java (revision 959954)
+++ src/test/org/apache/nutch/indexer/TestIndexSorter.java (working copy)
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nutch.indexer;
-
-import java.io.File;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriter.MaxFieldLength;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.nutch.analysis.NutchDocumentAnalyzer;
-import org.apache.nutch.util.NutchConfiguration;
-
-import junit.framework.TestCase;
-
-public class TestIndexSorter extends TestCase {
- private static final Log LOG = LogFactory.getLog(TestIndexSorter.class);
-
- private static final String INDEX_PLAIN = "index";
- private static final String INDEX_SORTED = "index-sorted";
- private static final int NUM_DOCS = 254;
- private String[] fieldNames = new String[] {
- "id",
- "url",
- "site",
- "content",
- "host",
- "anchor",
- "boost"
- };
-
- Configuration conf = null;
- File testDir = null;
- Directory dir = null;
-
-
- protected void setUp() throws Exception {
- if (conf == null) conf = NutchConfiguration.create();
- // create test index
- testDir = new File("indexSorter-test-" + System.currentTimeMillis());
- if (!testDir.mkdirs()) {
- throw new Exception("Can't create test dir " + testDir.toString());
- }
- LOG.info("Creating test index: " + testDir.getAbsolutePath());
- File plain = new File(testDir, INDEX_PLAIN);
- Directory dir = FSDirectory.open(plain);
- IndexWriter writer = new IndexWriter(dir, new NutchDocumentAnalyzer(conf), true,
- MaxFieldLength.UNLIMITED);
- // create test documents
- for (int i = 0; i < NUM_DOCS; i++) {
- Document doc = new Document();
- for (int k = 0; k < fieldNames.length; k++) {
- Field f;
- Store s;
- Index ix;
- String val = null;
- if (fieldNames[k].equals("id")) {
- s = Store.YES;
- ix = Index.NOT_ANALYZED;
- val = String.valueOf(i);
- } else if (fieldNames[k].equals("host")) {
- s = Store.YES;
- ix = Index.NOT_ANALYZED;
- val = "www.example" + i + ".com";
- } else if (fieldNames[k].equals("site")) {
- s = Store.NO;
- ix = Index.NOT_ANALYZED;
- val = "www.example" + i + ".com";
- } else if (fieldNames[k].equals("content")) {
- s = Store.NO;
- ix = Index.ANALYZED;
- val = "This is the content of the " + i + "-th document.";
- } else if (fieldNames[k].equals("boost")) {
- s = Store.YES;
- ix = Index.NO;
- // XXX note that this way we ensure different values of encoded boost
- // XXX note also that for this reason we can't reliably test more than
- // XXX 255 documents.
- float boost = Similarity.decodeNorm((byte)(i + 1));
- val = String.valueOf(boost);
- doc.setBoost(boost);
- } else {
- s = Store.YES;
- ix = Index.ANALYZED;
- if (fieldNames[k].equals("anchor")) {
- val = "anchors to " + i + "-th page.";
- } else if (fieldNames[k].equals("url")) {
- val = "http://www.example" + i + ".com/" + i + ".html";
- }
- }
- f = new Field(fieldNames[k], val, s, ix);
- doc.add(f);
- }
- writer.addDocument(doc);
- }
- writer.optimize();
- writer.close();
- }
-
- protected void tearDown() throws Exception {
- FileUtil.fullyDelete(testDir);
- }
-
- public void testSorting() throws Exception {
- IndexSorter sorter = new IndexSorter(conf);
- sorter.sort(testDir);
-
- // read back documents
- IndexReader reader = IndexReader.open(FSDirectory.open(new File(testDir, INDEX_SORTED)));
- assertEquals(reader.numDocs(), NUM_DOCS);
- for (int i = 0; i < reader.maxDoc(); i++) {
- Document doc = reader.document(i);
- Field f = doc.getField("content");
- assertNull(f);
- f = doc.getField("boost");
- float boost = Similarity.decodeNorm((byte)(NUM_DOCS - i));
- String cmp = String.valueOf(boost);
- assertEquals(cmp, f.stringValue());
- }
- reader.close();
- }
-
-}
Index: src/test/org/apache/nutch/ontology/TestOntologyFactory.java
===================================================================
--- src/test/org/apache/nutch/ontology/TestOntologyFactory.java (revision 959954)
+++ src/test/org/apache/nutch/ontology/TestOntologyFactory.java (working copy)
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nutch.ontology;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.plugin.PluginRuntimeException;
-import org.apache.nutch.util.NutchConfiguration;
-
-import junit.framework.TestCase;
-
-public class TestOntologyFactory extends TestCase {
-
- private Configuration conf;
-
- protected void setUp() throws Exception {
- conf = NutchConfiguration.create();
- conf.set("plugin.includes", ".*");
- }
-
- public void testGetOntology() {
- OntologyFactory factory=new OntologyFactory(conf);
-
- try {
- Ontology ontology1=factory.getOntology();
- Ontology ontology2=factory.getOntology();
- assertNotNull(ontology1);
- assertNotNull(ontology2);
- //Current implementation creates new object instance in every call
- //TODO: check if this is required
- assertNotSame(ontology1, ontology2);
- } catch (PluginRuntimeException e) {
- fail("should not trow:" + e);
- }
- }
-}
Index: src/java/org/apache/nutch/clustering/OnlineClustererFactory.java
===================================================================
--- src/java/org/apache/nutch/clustering/OnlineClustererFactory.java (revision 959954)
+++ src/java/org/apache/nutch/clustering/OnlineClustererFactory.java (working copy)
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nutch.clustering;
-
-// Commons Logging imports
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.nutch.plugin.*;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A factory for retrieving {@link OnlineClusterer} extensions.
- *
- * @author Dawid Weiss
- * @version $Id$
- */
-public class OnlineClustererFactory {
- public static final Log LOG = LogFactory.getLog(OnlineClustererFactory.class);
-
- /**
- * Nutch configuration key specifying a particular clustering extension
- * to use.
- */
- private final static String CONFIG_FIELD_NAME = "extension.clustering.extension-name";
-
- /**
- * An {@link ExtensionPoint} pointing to {@link OnlineClusterer}.
- */
- private ExtensionPoint extensionPoint;
-
- /**
- * Default clustering extension implementation retrieved from the
- * configuration file or null if the default (first encountered extension)
- * is to be used.
- */
- private String extensionName;
-
- /**
- * Create an instance of the clustering factory bound to
- * a given configuration.
- */
- public OnlineClustererFactory(Configuration conf) {
- this.extensionPoint = PluginRepository.get(conf).getExtensionPoint(OnlineClusterer.X_POINT_ID);
- this.extensionName = conf.get(CONFIG_FIELD_NAME);
- }
-
- /**
- * @return Returns the online clustering extension specified
- * in nutch configuration (key name is extension.clustering.extension-name).
- * If the name is empty (no preference), the first available clustering extension is
- * returned.
- */
- public OnlineClusterer getOnlineClusterer()
- throws PluginRuntimeException {
-
- if (this.extensionPoint == null) {
- // not even an extension point defined.
- return null;
- }
-
- if (extensionName != null) {
- Extension extension = findExtension(extensionName);
- if (extension != null) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Using clustering extension: " + extensionName);
- }
- return (OnlineClusterer) extension.getExtensionInstance();
- }
- if (LOG.isWarnEnabled()) {
- LOG.warn("Clustering extension not found: '" + extensionName +
- "', trying the default");
- }
- // not found, fallback to the default, if available.
- }
-
- final Extension[] extensions = this.extensionPoint.getExtensions();
- if (extensions.length > 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Using the first clustering extension found: " +
- extensions[0].getId());
- }
- return (OnlineClusterer) extensions[0].getExtensionInstance();
- } else {
- return null;
- }
- }
-
- private Extension findExtension(String name)
- throws PluginRuntimeException {
-
- final Extension[] extensions = this.extensionPoint.getExtensions();
- for (int i = 0; i < extensions.length; i++) {
- final Extension extension = extensions[i];
- if (name.equals(extension.getId()))
- return extension;
- }
- return null;
- }
-
-}
Index: src/java/org/apache/nutch/clustering/HitsCluster.java
===================================================================
--- src/java/org/apache/nutch/clustering/HitsCluster.java (revision 959954)
+++ src/java/org/apache/nutch/clustering/HitsCluster.java (working copy)
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nutch.clustering;
-
-import org.apache.nutch.searcher.HitDetails;
-
-/**
- * An interface representing a group (cluster) of related hits.
- *
- * If {@link #isJunkCluster()} method returns true
- * then this cluster contains documents that are grouped together,
- * but no clear semantic relation has been detected. Such clusters may
- * be hidden in the user interface layer, unless someone wishes to
- * see an explicit group of documents that didn't belong anywhere else.
null
- * if this cluster has no sub-groups.
- */
- public HitsCluster [] getSubclusters();
-
- /**
- * @return Returns a relevance-ordered array of the hits belonging
- * to this cluster or null if this cluster
- * has no associated documents (it may have subclusters only).
- */
- public HitDetails[] getHits();
-
- /**
- * @return Returns an array of labels for this cluster. The labels should
- * be sorted according to their relevance to the cluster's content. Not
- * all of the labels must be displayed - the application is free to
- * set a cutoff threshold and display only the topmost labels.
- */
- public String[] getDescriptionLabels();
-
- /**
- * Returns true if this cluster constains documents
- * that did not fit anywhere else (presentation layer may
- * discard such clusters).
- *
- * Subclusters of this cluster are also junk clusters, even if
- * they don't have this property set to true
By the term online search results clustering we will understand - * a clusterer that works on a set of {@link HitDetails} retrieved for a - * query and able to produce a set of {@link HitsCluster} that can be displayed - * to help the user gain more insight in the topics found in the result.
- * - *Other clustering options include predefined categories and off-line - * preclustered groups, but I do not investigate those any further here.
- * - * @author Dawid Weiss - * @version $Id$ - */ -public interface OnlineClusterer extends Pluggable { - /** The name of the extension point. */ - public final static String X_POINT_ID = OnlineClusterer.class.getName(); - - /** - * Clusters an array of hits ({@link HitDetails} objects) and - * their previously extracted summaries (Strings).
- *
- * Arguments to this method may seem to be very low-level, but - * in fact they are side products of a regular search process, - * so we simply reuse them instead of duplicating part of the usual - * Nutch functionality. Other ideas are welcome.
- * - *This method must be thread-safe (many threads may invoke - * it concurrently on the same instance of a clusterer).
- * - * @return A set of {@link HitsCluster} objects. - */ - public HitsCluster [] clusterHits(HitDetails [] hitDetails, String [] descriptions); -} Index: src/java/org/apache/nutch/analysis/TokenMgrError.java =================================================================== --- src/java/org/apache/nutch/analysis/TokenMgrError.java (revision 959954) +++ src/java/org/apache/nutch/analysis/TokenMgrError.java (working copy) @@ -1,134 +0,0 @@ -/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */ -package org.apache.nutch.analysis; - -@SuppressWarnings("serial") -class TokenMgrError extends Error -{ - /* - * Ordinals for various reasons why an Error of this type can be thrown. - */ - - /** - * Lexical error occured. - */ - static final int LEXICAL_ERROR = 0; - - /** - * An attempt wass made to create a second instance of a static token manager. - */ - static final int STATIC_LEXER_ERROR = 1; - - /** - * Tried to change to an invalid lexical state. - */ - static final int INVALID_LEXICAL_STATE = 2; - - /** - * Detected (and bailed out of) an infinite loop in the token manager. - */ - static final int LOOP_DETECTED = 3; - - /** - * Indicates the reason why the exception is thrown. It will have - * one of the above 4 values. - */ - int errorCode; - - /** - * Replaces unprintable characters by their espaced (or unicode escaped) - * equivalents in the given string - */ - protected static final String addEscapes(String str) { - StringBuffer retval = new StringBuffer(); - char ch; - for (int i = 0; i < str.length(); i++) { - switch (str.charAt(i)) - { - case 0 : - continue; - case '\b': - retval.append("\\b"); - continue; - case '\t': - retval.append("\\t"); - continue; - case '\n': - retval.append("\\n"); - continue; - case '\f': - retval.append("\\f"); - continue; - case '\r': - retval.append("\\r"); - continue; - case '\"': - retval.append("\\\""); - continue; - case '\'': - retval.append("\\\'"); - continue; - case '\\': - retval.append("\\\\"); - continue; - default: - if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { - String s = "0000" + Integer.toString(ch, 16); - retval.append("\\u" + s.substring(s.length() - 4, s.length())); - } else { - retval.append(ch); - } - continue; - } - } - return retval.toString(); - } - - /** - * Returns a detailed message for the Error when it is thrown by the - * token manager to indicate a lexical error. - * Parameters : - * EOFSeen : indicates if EOF caused the lexicl error - * curLexState : lexical state in which this error occured - * errorLine : line number when the error occured - * errorColumn : column number when the error occured - * errorAfter : prefix that was seen before this error occured - * curchar : the offending character - * Note: You can customize the lexical error message by modifying this method. - */ - protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { - return("Lexical error at line " + - errorLine + ", column " + - errorColumn + ". Encountered: " + - (EOFSeen ? "