Index: ivy/ivy.xml =================================================================== --- ivy/ivy.xml (revision 1484186) +++ ivy/ivy.xml (working copy) @@ -76,7 +76,7 @@ - + Index: src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java =================================================================== --- src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java (revision 1484186) +++ src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java (working copy) @@ -32,14 +32,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** * Unit tests for SWFParser. * * @author Andrzej Bialecki */ -public class TestSWFParser extends TestCase { +public class TestSWFParser { private String fileSeparator = System.getProperty("file.separator"); // This system property is defined in ./src/plugin/build-plugin.xml @@ -47,10 +48,8 @@ private String[] sampleFiles = new String[]{"test1.swf", "test2.swf", "test3.swf"}; private String[] sampleTexts = new String[]{"test1.txt", "test2.txt", "test3.txt"}; - private String[] texts = new String[sampleTexts.length]; - + public TestSWFParser(String name) { - super(name); for (int i = 0; i < sampleFiles.length; i++) { try { // read the test string @@ -74,6 +73,7 @@ protected void tearDown() {} + @Test public void testIt() throws ProtocolException, ParseException { String urlString; Protocol protocol; Index: src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java =================================================================== --- src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java (revision 1484186) +++ src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java (working copy) @@ -19,10 +19,9 @@ import java.io.IOException; import java.io.StringReader; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import junit.textui.TestRunner; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; /** @@ -30,7 +29,7 @@ * * @author Andrzej Bialecki */ -public class TestSuffixURLFilter extends TestCase { +public class TestSuffixURLFilter { private static final String suffixes = "# this is a comment\n" + "\n" + @@ -117,22 +116,12 @@ private SuffixURLFilter filter = null; - public TestSuffixURLFilter(String testName) { - super(testName); - } - - public static Test suite() { - return new TestSuite(TestSuffixURLFilter.class); - } - - public static void main(String[] args) { - TestRunner.run(suite()); - } - + @Before public void setUp() throws IOException { filter = new SuffixURLFilter(new StringReader(suffixes)); } + @Test public void testModeAccept() { filter.setIgnoreCase(false); filter.setModeAccept(true); @@ -141,6 +130,7 @@ } } + @Test public void testModeReject() { filter.setIgnoreCase(false); filter.setModeAccept(false); @@ -149,6 +139,7 @@ } } + @Test public void testModeAcceptIgnoreCase() { filter.setIgnoreCase(true); filter.setModeAccept(true); @@ -157,6 +148,7 @@ } } + @Test public void testModeRejectIgnoreCase() { filter.setIgnoreCase(true); filter.setModeAccept(false); @@ -165,6 +157,7 @@ } } + @Test public void testModeAcceptAndNonPathFilter() { filter.setModeAccept(true); filter.setFilterFromPath(false); @@ -173,6 +166,7 @@ } } + @Test public void testModeAcceptAndPathFilter() { filter.setModeAccept(true); filter.setFilterFromPath(true); Index: src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java =================================================================== --- src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java (revision 1484186) +++ src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java (working copy) @@ -29,15 +29,16 @@ import org.apache.nutch.util.NutchConfiguration; import org.apache.hadoop.io.Text; +import org.junit.Test; -import junit.framework.TestCase; +import static org.junit.Assert.*; /** * Based on Unit tests for MSWordParser by John Xing * * @author Rohit Kulkarni & Ashish Vaidya */ -public class TestZipParser extends TestCase { +public class TestZipParser { private String fileSeparator = System.getProperty("file.separator"); // This system property is defined in ./src/plugin/build-plugin.xml @@ -50,13 +51,13 @@ private String expectedText = "textfile.txt This is text file number 1 "; public TestZipParser(String name) { - super(name); } protected void setUp() {} protected void tearDown() {} + @Test public void testIt() throws ProtocolException, ParseException { String urlString; Protocol protocol; Index: src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java (revision 1484186) +++ src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java (working copy) @@ -24,22 +24,22 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.Reader; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; - /** Unit tests for RegexUrlNormalizer. */ -public class TestRegexURLNormalizer extends TestCase { +public class TestRegexURLNormalizer { private static final Logger LOG = LoggerFactory.getLogger(TestRegexURLNormalizer.class); private RegexURLNormalizer normalizer; @@ -51,8 +51,8 @@ // Make sure sample files are copied to "test.data" as specified in // ./src/plugin/urlnormalizer-regex/build.xml during plugin compilation. - public TestRegexURLNormalizer(String name) throws IOException { - super(name); + @Before + public void setUp() throws IOException { normalizer = new RegexURLNormalizer(); conf = NutchConfiguration.create(); normalizer.setConf(conf); @@ -77,11 +77,13 @@ } } + @Test public void testNormalizerDefault() throws Exception { normalizeTest((NormalizedURL[])testData.get(URLNormalizers.SCOPE_DEFAULT), URLNormalizers.SCOPE_DEFAULT); } + @Test public void testNormalizerScope() throws Exception { Iterator it = testData.keySet().iterator(); while (it.hasNext()) { @@ -139,43 +141,4 @@ } return (NormalizedURL[]) list.toArray(new NormalizedURL[list.size()]); } - - public static void main(String[] args) throws Exception { - if (args.length == 0) { - System.err.println("TestRegexURLNormalizer [-bench ] "); - System.exit(-1); - } - boolean bench = false; - int iter = -1; - String scope = null; - for (int i = 0; i < args.length; i++) { - if (args[i].equals("-bench")) { - bench = true; - iter = Integer.parseInt(args[++i]); - } else scope = args[i]; - } - if (scope == null) { - System.err.println("Missing required scope name."); - System.exit(-1); - } - if (bench && iter < 0) { - System.err.println("Invalid number of iterations: " + iter); - System.exit(-1); - } - TestRegexURLNormalizer test = new TestRegexURLNormalizer("test"); - NormalizedURL[] urls = (NormalizedURL[])test.testData.get(scope); - if (urls == null) { - LOG.warn("Missing test data for scope '" + scope + "', using default scope."); - scope = URLNormalizers.SCOPE_DEFAULT; - urls = (NormalizedURL[])test.testData.get(scope); - } - if (bench) { - test.bench(iter, scope); - } else { - test.normalizeTest(urls, scope); - } - } - - - } Index: src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java =================================================================== --- src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java (revision 1484186) +++ src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java (working copy) @@ -20,14 +20,12 @@ import java.io.IOException; import java.io.Reader; -// JUnit imports -import junit.framework.Test; -import junit.framework.TestSuite; -import junit.textui.TestRunner; import org.apache.nutch.net.*; // Nutch imports import org.apache.nutch.urlfilter.api.RegexURLFilterBaseTest; +import org.junit.Test; +import static org.junit.Assert.*; /** @@ -37,18 +35,6 @@ */ public class TestAutomatonURLFilter extends RegexURLFilterBaseTest { - public TestAutomatonURLFilter(String testName) { - super(testName); - } - - public static Test suite() { - return new TestSuite(TestAutomatonURLFilter.class); - } - - public static void main(String[] args) { - TestRunner.run(suite()); - } - protected URLFilter getURLFilter(Reader rules) { try { return new AutomatonURLFilter(rules); @@ -58,6 +44,7 @@ } } + @Test public void test() { test("WholeWebCrawling"); test("IntranetCrawling"); Index: src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java =================================================================== --- src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java (revision 1484186) +++ src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java (working copy) @@ -18,10 +18,8 @@ package org.apache.nutch.protocol.file; // Hadoop imports -import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.protocol.Protocol; import org.apache.nutch.protocol.ProtocolException; @@ -31,6 +29,9 @@ import org.apache.nutch.protocol.ProtocolStatusCodes; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; /** * @author mattmann @@ -40,7 +41,7 @@ * Unit tests for the {@link File}Protocol. *

. */ -public class TestProtocolFile extends TestCase { +public class TestProtocolFile { private String fileSeparator = System.getProperty("file.separator"); private String sampleDir = System.getProperty("test.data", "."); @@ -52,10 +53,12 @@ private Configuration conf; - protected void setUp() { + @Before + public void setUp() { conf = NutchConfiguration.create(); } + @Test public void testSetContentType() throws ProtocolException { for (String testTextFile : testTextFiles) { setContentType(testTextFile); Index: src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java =================================================================== --- src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java (revision 1484186) +++ src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java (working copy) @@ -20,15 +20,11 @@ import java.io.IOException; import java.io.Reader; -// JUnit imports -import junit.framework.Test; -import junit.framework.TestSuite; -import junit.textui.TestRunner; import org.apache.nutch.net.*; - -// Nutch imports import org.apache.nutch.urlfilter.api.RegexURLFilterBaseTest; +import org.junit.Test; +import static org.junit.Assert.*; /** * JUnit based test of class RegexURLFilter. @@ -37,18 +33,6 @@ */ public class TestRegexURLFilter extends RegexURLFilterBaseTest { - public TestRegexURLFilter(String testName) { - super(testName); - } - - public static Test suite() { - return new TestSuite(TestRegexURLFilter.class); - } - - public static void main(String[] args) { - TestRunner.run(suite()); - } - protected URLFilter getURLFilter(Reader rules) { try { return new RegexURLFilter(rules); @@ -58,6 +42,7 @@ } } + @Test public void test() { test("WholeWebCrawling"); test("IntranetCrawling"); Index: src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java =================================================================== --- src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java (revision 1484186) +++ src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java (working copy) @@ -17,16 +17,20 @@ package org.apache.nutch.protocol.http.api; +import org.junit.Before; +import org.junit.Test; + import crawlercommons.robots.BaseRobotRules; -import junit.framework.TestCase; +import static org.junit.Assert.*; + /** * JUnit test case which tests * 1. that robots filtering is performed correctly as per the agent name * 2. that crawl delay is extracted correctly from the robots file * */ -public class TestRobotRulesParser extends TestCase { +public class TestRobotRulesParser { private static final String CONTENT_TYPE = "text/plain"; private static final String SINGLE_AGENT = "Agent1"; @@ -72,14 +76,15 @@ private HttpRobotRulesParser parser; private BaseRobotRules rules; - public TestRobotRulesParser(String name) { - super(name); + @Before + public void setUp() { parser = new HttpRobotRulesParser(); } /** * Test that the robots rules are interpreted correctly by the robots rules parser. */ + @Test public void testRobotsAgent() { rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(), CONTENT_TYPE, SINGLE_AGENT); @@ -104,6 +109,7 @@ * Test that the crawl delay is extracted from the robots file for respective agent. * If its not specified for a given agent, default value must be returned. */ + @Test public void testCrawlDelay() { // for SINGLE_AGENT, the crawl delay of 10 sec ie. 10000 msec must be returned by the parser rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(), CONTENT_TYPE, SINGLE_AGENT); Index: src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java =================================================================== --- src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java (revision 1484186) +++ src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java (working copy) @@ -17,29 +17,24 @@ package org.creativecommons.nutch; -import org.apache.nutch.metadata.Metadata; -import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseUtil; -import org.apache.nutch.protocol.Content; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.tika.mime.MimeType; - -import java.util.Properties; import java.io.*; -import java.net.URL; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestCCParseFilter extends TestCase { +public class TestCCParseFilter { private static final File testDir = new File( System.getProperty("test.input")); + @Test public void testPages() throws Exception { pageTest(new File(testDir, "anchor.html"), "http://foo.com/", "http://creativecommons.org/licenses/by-nc-sa/1.0", "a", null); @@ -56,7 +51,6 @@ public void pageTest(File file, String url, String license, String location, String type) throws Exception { - String contentType = "text/html"; InputStream in = new FileInputStream(file); ByteArrayOutputStream out = new ByteArrayOutputStream( (int) file.length()); Index: src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java =================================================================== --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java (revision 1484186) +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java (working copy) @@ -23,9 +23,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.nutch.protocol.ProtocolFactory; -import org.apache.nutch.protocol.Protocol; -import org.apache.nutch.protocol.Content; import org.apache.nutch.protocol.ProtocolException; import org.apache.nutch.parse.Parse; @@ -38,17 +35,16 @@ import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.tika.mime.MimeType; +import org.junit.Test; +import static org.junit.Assert.*; -import junit.framework.TestCase; - /** * Unit tests for the RSS Parser based on John Xing's TestPdfParser class. * * @author mattmann * @version 1.0 */ -public class TestRSSParser extends TestCase { +public class TestRSSParser { private String fileSeparator = System.getProperty("file.separator"); @@ -62,18 +58,6 @@ /** *

- * Default constructor - *

- * - * @param name - * The name of the RSSParserTest - */ - public TestRSSParser(String name) { - super(name); - } - - /** - *

* The test method: tests out the following 2 asserts: *

* @@ -83,9 +67,9 @@ * file * */ + @Test public void testIt()throws ProtocolException, ParseException, IOException { String urlString; - Protocol protocol; Parse parse; Configuration conf = NutchConfiguration.create(); Index: src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java =================================================================== --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java (revision 1484186) +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java (working copy) @@ -23,7 +23,8 @@ import java.io.IOException; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -34,14 +35,13 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.tika.mime.MimeType; /** * Unit tests for PdfParser. * * @author John Xing */ -public class TestPdfParser extends TestCase { +public class TestPdfParser { private String fileSeparator = System.getProperty("file.separator"); // This system property is defined in ./src/plugin/build-plugin.xml @@ -53,16 +53,7 @@ private String expectedText = "A VERY SMALL PDF FILE"; - public TestPdfParser(String name) { - super(name); - } - - protected void setUp() { - } - - protected void tearDown() { - } - + @Test public void testIt() throws ProtocolException, ParseException, IOException { String urlString; Parse parse; Index: src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java =================================================================== --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java (revision 1484186) +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java (working copy) @@ -23,7 +23,9 @@ import java.io.IOException; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -34,14 +36,13 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.tika.mime.MimeType; /** * Unit tests for MSWordParser. * * @author John Xing */ -public class TestMSWordParser extends TestCase { +public class TestMSWordParser { private String fileSeparator = System.getProperty("file.separator"); // This system property is defined in ./src/plugin/build-plugin.xml @@ -55,18 +56,12 @@ private Configuration conf; - public TestMSWordParser(String name) { - super(name); - } - - protected void setUp() { + @Before + public void setUp() { conf = NutchConfiguration.create(); conf.set("file.content.limit", "-1"); } - protected void tearDown() { - } - public String getTextContent(String fileName) throws ProtocolException, ParseException, IOException { String urlString = sampleDir + fileSeparator + fileName; @@ -89,6 +84,7 @@ return parse.getText(); } + @Test public void testIt() throws ProtocolException, ParseException, IOException { for (int i = 0; i < sampleFiles.length; i++) { String found = getTextContent(sampleFiles[i]); @@ -97,6 +93,7 @@ } } + @Test public void testOpeningDocs() throws ProtocolException, ParseException, IOException { String[] filenames = new File(sampleDir).list(); for (int i = 0; i < filenames.length; i++) { Index: src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java =================================================================== --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java (revision 1484186) +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java (working copy) @@ -23,7 +23,8 @@ import java.io.IOException; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -34,14 +35,13 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.tika.mime.MimeType; /** * Unit tests for TestRTFParser. (Adapted from John Xing msword unit tests). * * @author Andy Hedges */ -public class TestRTFParser extends TestCase { +public class TestRTFParser { private String fileSeparator = System.getProperty("file.separator"); // This system property is defined in ./src/plugin/build-plugin.xml @@ -51,16 +51,7 @@ // Check ./src/plugin/parse-rtf/sample/README.txt for what they are. private String rtfFile = "test.rtf"; - public TestRTFParser(String name) { - super(name); - } - - protected void setUp() { - } - - protected void tearDown() { - } - + @Test public void testIt() throws ProtocolException, ParseException, IOException { String urlString; Index: src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java =================================================================== --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java (revision 1484186) +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java (working copy) @@ -24,7 +24,8 @@ import java.io.InputStreamReader; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -35,92 +36,83 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; -import org.apache.tika.mime.MimeType; /** * Unit tests for OOParser. * * @author Andrzej Bialecki */ -public class TestOOParser extends TestCase { +public class TestOOParser { - private String fileSeparator = System.getProperty("file.separator"); - // This system property is defined in ./src/plugin/build-plugin.xml - private String sampleDir = System.getProperty("test.data", "."); - // Make sure sample files are copied to "test.data" as specified in - // ./src/plugin/parse-oo/build.xml during plugin compilation. - private String[] sampleFiles = { "ootest.odt", "ootest.sxw" }; + private String fileSeparator = System.getProperty("file.separator"); + // This system property is defined in ./src/plugin/build-plugin.xml + private String sampleDir = System.getProperty("test.data", "."); + // Make sure sample files are copied to "test.data" as specified in + // ./src/plugin/parse-oo/build.xml during plugin compilation. + private String[] sampleFiles = { "ootest.odt", "ootest.sxw" }; - private String sampleText = "ootest.txt"; + private String sampleText = "ootest.txt"; - private String expectedText; + private String expectedText; - public TestOOParser(String name) { - super(name); - try { - // read the test string - FileInputStream fis = new FileInputStream(sampleDir + fileSeparator - + sampleText); - StringBuffer sb = new StringBuffer(); - int len = 0; - InputStreamReader isr = new InputStreamReader(fis, "UTF-8"); - char[] buf = new char[1024]; - while ((len = isr.read(buf)) > 0) { - sb.append(buf, 0, len); - } - isr.close(); - expectedText = sb.toString(); - // normalize space - expectedText = expectedText.replaceAll("[ \t\r\n]+", " "); - } catch (Exception e) { - e.printStackTrace(); - } + @Test + public void testIt() throws ProtocolException, ParseException, IOException { + String urlString; + Parse parse; + Configuration conf = NutchConfiguration.create(); + MimeUtil mimeutil = new MimeUtil(conf); + + try { + // read the test string + FileInputStream fis = new FileInputStream(sampleDir + fileSeparator + + sampleText); + StringBuffer sb = new StringBuffer(); + int len = 0; + InputStreamReader isr = new InputStreamReader(fis, "UTF-8"); + char[] buf = new char[1024]; + while ((len = isr.read(buf)) > 0) { + sb.append(buf, 0, len); + } + isr.close(); + expectedText = sb.toString(); + // normalize space + expectedText = expectedText.replaceAll("[ \t\r\n]+", " "); + } catch (Exception e) { + e.printStackTrace(); } - protected void setUp() { - } + System.out.println("Expected : " + expectedText); - protected void tearDown() { - } + for (int i = 0; i < sampleFiles.length; i++) { + urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; - public void testIt() throws ProtocolException, ParseException, IOException { - String urlString; - Parse parse; - Configuration conf = NutchConfiguration.create(); - MimeUtil mimeutil = new MimeUtil(conf); + if (sampleFiles[i].startsWith("ootest") == false) + continue; - System.out.println("Expected : " + expectedText); + File file = new File(sampleDir + fileSeparator + sampleFiles[i]); + byte[] bytes = new byte[(int) file.length()]; + DataInputStream in = new DataInputStream(new FileInputStream(file)); + in.readFully(bytes); + in.close(); - for (int i = 0; i < sampleFiles.length; i++) { - urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i]; + WebPage page = new WebPage(); + page.setBaseUrl(new Utf8(urlString)); + page.setContent(ByteBuffer.wrap(bytes)); + String mtype = mimeutil.getMimeType(file); + page.setContentType(new Utf8(mtype)); - if (sampleFiles[i].startsWith("ootest") == false) - continue; + parse = new ParseUtil(conf).parse(urlString, page); - File file = new File(sampleDir + fileSeparator + sampleFiles[i]); - byte[] bytes = new byte[(int) file.length()]; - DataInputStream in = new DataInputStream(new FileInputStream(file)); - in.readFully(bytes); - in.close(); + String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim(); - WebPage page = new WebPage(); - page.setBaseUrl(new Utf8(urlString)); - page.setContent(ByteBuffer.wrap(bytes)); - String mtype = mimeutil.getMimeType(file); - page.setContentType(new Utf8(mtype)); + // simply test for the presence of a text - the ordering of the + // elements + // may differ from what was expected + // in the previous tests + assertTrue(text != null && text.length() > 0); - parse = new ParseUtil(conf).parse(urlString, page); - - String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim(); - - // simply test for the presence of a text - the ordering of the - // elements - // may differ from what was expected - // in the previous tests - assertTrue(text != null && text.length() > 0); - - System.out.println("Found " + sampleFiles[i] + ": " + text); - } + System.out.println("Found " + sampleFiles[i] + ": " + text); } + } } Index: src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java =================================================================== --- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java (revision 1484186) +++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java (working copy) @@ -17,7 +17,8 @@ package org.apache.nutch.parse.tika; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.nutch.parse.Outlink; import org.apache.nutch.parse.tika.DOMBuilder; @@ -43,7 +44,7 @@ /** * Unit tests for DOMContentUtils. */ -public class DOMContentUtilsTest extends TestCase { +public class DOMContentUtilsTest { private static final String[] testPages = { // 0. @@ -200,7 +201,6 @@ public static final Logger Logger = LoggerFactory.getLogger(DOMContentUtilsTest.class); public DOMContentUtilsTest(String name) { - super(name); } private static void setup() throws Exception { @@ -317,6 +317,7 @@ return true; } + @Test public void testGetText() throws Exception { if (testDOMs[0] == null) setup(); @@ -335,6 +336,7 @@ // won't work with Tika - the title is stored in the metadata but // not put in the XHTML representation + @Test public void testGetTitle() throws Exception { if (testDOMs[0] == null) setup(); @@ -351,6 +353,7 @@ } } + @Test public void testGetOutlinks() throws Exception { if (testDOMs[0] == null) setup(); Index: src/plugin/lib-regex-filter/plugin.xml =================================================================== --- src/plugin/lib-regex-filter/plugin.xml (revision 1484186) +++ src/plugin/lib-regex-filter/plugin.xml (working copy) @@ -28,6 +28,8 @@ + + Index: src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java =================================================================== --- src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java (revision 1484186) +++ src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java (working copy) @@ -24,23 +24,30 @@ import java.util.ArrayList; import java.util.List; -// JUnit imports -import junit.framework.TestCase; +import static org.junit.Assert.*; -// Commons Logging imports +// Logging imports import org.slf4j.Logger; import org.slf4j.LoggerFactory; // Nutch imports import org.apache.nutch.net.URLFilter; +//import org.apache.nutch.urlfilter.automaton.TestAutomatonURLFilter; +//import org.apache.nutch.urlfilter.regex.TestRegexURLFilter; +//import org.junit.runners.Suite; +//import org.junit.runner.RunWith; /** * JUnit based test of class RegexURLFilterBase. * * @author Jérôme Charron */ -public abstract class RegexURLFilterBaseTest extends TestCase { + + +//@RunWith(Suite.class) +//@Suite.SuiteClasses({TestAutomatonURLFilter.class, TestRegexURLFilter.class}) +public abstract class RegexURLFilterBaseTest { /** My logger */ protected static final Logger LOG = LoggerFactory.getLogger(RegexURLFilterBaseTest.class); @@ -48,10 +55,6 @@ private final static String SEPARATOR = System.getProperty("file.separator"); private final static String SAMPLES = System.getProperty("test.data", "."); - public RegexURLFilterBaseTest(String testName) { - super(testName); - } - protected abstract URLFilter getURLFilter(Reader rules); protected void bench(int loops, String file) { Index: src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java =================================================================== --- src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java (revision 1484186) +++ src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java (working copy) @@ -35,8 +35,8 @@ import org.apache.nutch.protocol.ProtocolNotFound; import org.apache.nutch.util.NutchConfiguration; -// Junit imports -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** * @@ -45,7 +45,7 @@ * Test Suite for the {@link FeedParser}. * */ -public class TestFeedParser extends TestCase { +public class TestFeedParser { private String fileSeparator = System.getProperty("file.separator"); @@ -67,7 +67,6 @@ * The name of this {@link TestCase}. */ public TestFeedParser(String name) { - super(name); } /** @@ -85,6 +84,7 @@ * @throws ParseException * If the {@link Parser}Layer cannot be loaded. */ + @Test public void testParseFetchChannel() throws ProtocolNotFound, ParseException { String urlString; Protocol protocol; Index: src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java =================================================================== --- src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java (revision 1484186) +++ src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java (working copy) @@ -21,7 +21,8 @@ import java.io.InputStreamReader; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.nutch.metadata.Metadata; @@ -32,7 +33,7 @@ import org.apache.nutch.util.NutchConfiguration; import org.apache.tika.language.LanguageIdentifier; -public class TestHTMLLanguageParser extends TestCase { +public class TestHTMLLanguageParser { private static Utf8 URL = new Utf8("http://foo.bar/"); @@ -49,6 +50,7 @@ /** * Test parsing of language identifiers from html **/ + @Test public void testMetaHTMLParsing() { try { @@ -71,6 +73,7 @@ } /** Test of LanguageParser.parseLanguage(String) method. */ + @Test public void testParseLanguage() { String tests[][] = { { "(SCHEME=ISO.639-1) sv", "sv" }, { "(SCHEME=RFC1766) sv-FI", "sv" }, { "(SCHEME=Z39.53) SWE", "sv" }, @@ -98,6 +101,7 @@ } } + @Test public void testLanguageIndentifier() { try { long total = 0; Index: src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java =================================================================== --- src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java (revision 1484186) +++ src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java (working copy) @@ -22,14 +22,16 @@ import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestSubcollection extends TestCase { +public class TestSubcollection { /**Test filtering logic * * @throws Exception */ + @Test public void testFilter() throws Exception { Subcollection sc=new Subcollection(NutchConfiguration.create()); sc.setWhiteList("www.nutch.org\nwww.apache.org"); @@ -46,6 +48,7 @@ assertEquals(null, sc.filter("http://www.google.com/")); } + @Test public void testInput(){ StringBuffer xml=new StringBuffer(); xml.append(""); Index: src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java =================================================================== --- src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java (revision 1484186) +++ src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java (working copy) @@ -18,7 +18,8 @@ import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -28,8 +29,9 @@ import org.apache.nutch.util.EncodingDetector; import org.apache.nutch.util.NutchConfiguration; -public class TestMoreIndexingFilter extends TestCase { +public class TestMoreIndexingFilter { + @Test public void testContentType() throws IndexingException { Configuration conf = NutchConfiguration.create(); assertContentType(conf, "text/html", "text/html"); @@ -45,6 +47,7 @@ /** * @since NUTCH-901 */ + @Test public void testNoParts(){ Configuration conf = NutchConfiguration.create(); conf.setBoolean("moreIndexingFilter.indexMimeTypeParts", false); Index: src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java =================================================================== --- src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java (revision 1484186) +++ src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java (working copy) @@ -30,7 +30,8 @@ import org.apache.hadoop.io.Text; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import java.io.File; import java.io.FileOutputStream; @@ -47,7 +48,7 @@ * * @author John Xing */ -public class TestExtParser extends TestCase { +public class TestExtParser { private File tempFile = null; private String urlString = null; private Content content = null; @@ -58,7 +59,6 @@ private String expectedMD5sum = "df46711a1a48caafc98b1c3b83aa1526"; public TestExtParser(String name) { - super(name); } protected void setUp() throws ProtocolException, IOException { @@ -95,6 +95,7 @@ // tempFile.delete(); } + @Test public void testIt() throws ParseException { String contentType; Index: src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java (revision 1484186) +++ src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java (working copy) @@ -23,10 +23,12 @@ import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestPassURLNormalizer extends TestCase { +public class TestPassURLNormalizer { + @Test public void testPassURLNormalizer() { Configuration conf = NutchConfiguration.create(); Index: src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java =================================================================== --- src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java (revision 1484186) +++ src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java (working copy) @@ -17,8 +17,6 @@ package org.apache.nutch.parse.html; -import junit.framework.TestCase; - import org.apache.nutch.parse.HTMLMetaTags; import java.io.ByteArrayInputStream; @@ -29,11 +27,11 @@ import org.w3c.dom.*; import org.apache.html.dom.*; +import org.junit.Test; +import static org.junit.Assert.*; + /** Unit tests for HTMLMetaProcessor. */ -public class TestRobotsMetaProcessor extends TestCase { - public TestRobotsMetaProcessor(String name) { - super(name); - } +public class TestRobotsMetaProcessor { /* @@ -126,6 +124,7 @@ private URL[][] currURLsAndAnswers; + @Test public void testRobotsMetaProcessor() { DOMFragmentParser parser= new DOMFragmentParser();; Index: src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java =================================================================== --- src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java (revision 1484186) +++ src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java (working copy) @@ -17,8 +17,6 @@ package org.apache.nutch.parse.html; -import junit.framework.TestCase; - import org.apache.nutch.parse.Outlink; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; @@ -34,10 +32,14 @@ import org.w3c.dom.*; import org.apache.html.dom.*; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + /** * Unit tests for DOMContentUtils. */ -public class TestDOMContentUtils extends TestCase { +public class TestDOMContentUtils { private static final String[] testPages= { new String(" title " @@ -215,11 +217,8 @@ private static Configuration conf; private static DOMContentUtils utils = null; - public TestDOMContentUtils(String name) { - super(name); - } - - private static void setup() { + @Before + public void setup() { conf = NutchConfiguration.create(); conf.setBoolean("parser.html.form.use_action", true); utils = new DOMContentUtils(conf); @@ -313,6 +312,7 @@ return true; } + @Test public void testGetText() { if (testDOMs[0] == null) setup(); @@ -328,6 +328,7 @@ } } + @Test public void testGetTitle() { if (testDOMs[0] == null) setup(); @@ -343,6 +344,7 @@ } } + @Test public void testGetOutlinks() { if (testDOMs[0] == null) setup(); Index: src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java =================================================================== --- src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java (revision 1484186) +++ src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java (working copy) @@ -20,7 +20,10 @@ import java.net.MalformedURLException; import java.net.URL; -import junit.framework.TestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; @@ -35,7 +38,7 @@ * * @author Susam Pal */ -public class TestProtocolHttpClient extends TestCase { +public class TestProtocolHttpClient { private Server server; private Configuration conf; @@ -43,7 +46,8 @@ private int port; private Http http = new Http(); - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { server = new Server(); @@ -69,7 +73,8 @@ http.setConf(conf); } - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { server.stop(); } @@ -79,6 +84,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testCookies() throws Exception { startServer(47500); fetchPage("/cookies.jsp", 200); @@ -92,6 +98,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testNoPreemptiveAuth() throws Exception { startServer(47500); fetchPage("/noauth.jsp", 200); @@ -104,6 +111,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testDefaultCredentials() throws Exception { startServer(47502); fetchPage("/basic.jsp", 200); @@ -116,6 +124,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testBasicAuth() throws Exception { startServer(47500); fetchPage("/basic.jsp", 200); @@ -132,6 +141,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testOtherRealmsNoAuth() throws Exception { startServer(47501); fetchPage("/basic.jsp", 200); @@ -146,6 +156,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testDigestAuth() throws Exception { startServer(47500); fetchPage("/digest.jsp", 200); @@ -158,6 +169,7 @@ * @throws Exception * If an error occurs or the test case fails. */ + @Test public void testNtlmAuth() throws Exception { startServer(47501); fetchPage("/ntlm.jsp", 200); Index: src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java =================================================================== --- src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java (revision 1484186) +++ src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java (working copy) @@ -16,25 +16,22 @@ */ package org.apache.nutch.urlfilter.domain; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -public class TestDomainURLFilter - extends TestCase { +public class TestDomainURLFilter { protected static final Logger LOG = LoggerFactory.getLogger(TestDomainURLFilter.class); private final static String SEPARATOR = System.getProperty("file.separator"); private final static String SAMPLES = System.getProperty("test.data", "."); - public TestDomainURLFilter(String testName) { - super(testName); - } - + @Test public void testFilter() throws Exception { Index: src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java =================================================================== --- src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java (revision 1484186) +++ src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java (working copy) @@ -16,14 +16,13 @@ */ package org.apache.nutch.indexer.anchor; -import junit.framework.TestCase; - import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.indexer.NutchDocument; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; import org.junit.Test; +import static org.junit.Assert.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +34,7 @@ * @author lewismc * */ -public class TestAnchorIndexingFilter extends TestCase { +public class TestAnchorIndexingFilter { public static final Logger LOG = LoggerFactory.getLogger(TestAnchorIndexingFilter.class); Index: src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java =================================================================== --- src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java (revision 1484186) +++ src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java (working copy) @@ -24,9 +24,8 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; import org.junit.Test; +import static org.junit.Assert.*; -import junit.framework.TestCase; - /** *JUnit test case for {@link RelTagIndexingFilter} which *simply asserts that a 'tag' field is obtained by the filter. @@ -34,7 +33,7 @@ *@author lewismc */ - public class TestRelTagIndexingFilter extends TestCase { + public class TestRelTagIndexingFilter { @Test public void testRelTagFields() throws Exception { Index: src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java =================================================================== --- src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java (revision 1484186) +++ src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java (working copy) @@ -32,20 +32,20 @@ import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; import org.junit.Test; -import junit.framework.TestCase; +import static org.junit.Assert.*; /** * Junit test for {@link RelTagParser} based mainly John Xing's parser tests. * We are not concerned with actual parse text within the sample file, instead * we assert that the rel-tags we expect are found in the WebPage metadata. * To check the parser is working as expected we unwrap the ByteBuffer obtained - * from metadata, the same type as * we use in expected (String). So just the + * from metadata, the same type as we use in expected (String). So just the * other way around as we wrapped the metadata value. * * @author lewismc * */ -public class TestRelTagParser extends TestCase { +public class TestRelTagParser { private String fileSeparator = System.getProperty("file.separator"); @@ -61,39 +61,34 @@ private Configuration conf; - public TestRelTagParser(String name) { - super(name); - } - @Test - public void testRelTagParser() throws ProtocolException, ParseException, IOException { - conf = NutchConfiguration.create(); - conf.set("file.content.limit", "-1"); - Parse parse; - String urlString = "file:" + sampleDir + fileSeparator + sampleFile; + public void testRelTagParser() throws ParseException, ProtocolException, IOException { + conf = NutchConfiguration.create(); + conf.set("file.content.limit", "-1"); + Parse parse; + String urlString = "file:" + sampleDir + fileSeparator + sampleFile; - File file = new File(sampleDir + fileSeparator + sampleFile); - byte[] bytes = new byte[(int) file.length()]; - DataInputStream in = new DataInputStream(new FileInputStream(file)); - in.readFully(bytes); - in.close(); + File file = new File(sampleDir + fileSeparator + sampleFile); + byte[] bytes = new byte[(int) file.length()]; + DataInputStream in = new DataInputStream(new FileInputStream(file)); + in.readFully(bytes); + in.close(); - WebPage page = new WebPage(); - page.setBaseUrl(new Utf8(urlString)); - page.setContent(ByteBuffer.wrap(bytes)); - MimeUtil mimeutil = new MimeUtil(conf); - String mtype = mimeutil.getMimeType(file); - page.setContentType(new Utf8(mtype)); - parse = new ParseUtil(conf).parse(urlString, page); - - //begin assertion for tests - ByteBuffer bbuf = page.getFromMetadata(new Utf8("Rel-Tag")); - byte[] byteArray = new byte[bbuf.remaining()]; - bbuf.get(byteArray); - String s = new String(byteArray); - //bbuf.flip(); - assertEquals("We expect 2 tab-separated rel-tag's extracted by the filter", - expectedRelTags, s); + WebPage page = new WebPage(); + page.setBaseUrl(new Utf8(urlString)); + page.setContent(ByteBuffer.wrap(bytes)); + MimeUtil mimeutil = new MimeUtil(conf); + String mtype = mimeutil.getMimeType(file); + page.setContentType(new Utf8(mtype)); + parse = new ParseUtil(conf).parse(urlString, page); + //begin assertion for tests + ByteBuffer bbuf = page.getFromMetadata(new Utf8("Rel-Tag")); + byte[] byteArray = new byte[bbuf.remaining()]; + bbuf.get(byteArray); + String s = new String(byteArray); + //bbuf.flip(); + assertEquals("We expect 2 tab-separated rel-tag's extracted by the filter", + expectedRelTags, s); } } \ No newline at end of file Index: src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java =================================================================== --- src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java (revision 1484186) +++ src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java (working copy) @@ -19,7 +19,6 @@ import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; @@ -33,10 +32,10 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.MimeUtil; import org.apache.nutch.util.NutchConfiguration; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.*; -import junit.framework.TestCase; - /** * JUnit test case for {@link JSParseFilter} which tests * 1. That 5 outlinks are extracted from JavaScript snippets embedded in HTML @@ -45,7 +44,7 @@ * @author lewismc */ -public class TestJSParseFilter extends TestCase { +public class TestJSParseFilter { private String fileSeparator = System.getProperty("file.separator"); @@ -58,50 +57,44 @@ private Configuration conf; - public TestJSParseFilter(String name) { - super(name); + @Before + public void setUp() { + conf = NutchConfiguration.create(); + conf.set("file.content.limit", "-1"); } - - protected void setUp() { - conf = NutchConfiguration.create(); - conf.set("file.content.limit", "-1"); - } - protected void tearDown() { - } - public Outlink[] getOutlinks(String[] sampleFiles) throws ProtocolException, ParseException, IOException { - String urlString; - Parse parse; + String urlString; + Parse parse; - urlString = "file:" + sampleDir + fileSeparator + sampleFiles; - File file = new File(urlString); - byte[] bytes = new byte[(int) file.length()]; - DataInputStream dip = new DataInputStream(new FileInputStream(file)); - dip.readFully(bytes); - dip.close(); + urlString = "file:" + sampleDir + fileSeparator + sampleFiles; + File file = new File(urlString); + byte[] bytes = new byte[(int) file.length()]; + DataInputStream dip = new DataInputStream(new FileInputStream(file)); + dip.readFully(bytes); + dip.close(); - WebPage page = new WebPage(); - page.setBaseUrl(new Utf8(urlString)); - page.setContent(ByteBuffer.wrap(bytes)); - MimeUtil mutil = new MimeUtil(conf); - String mime = mutil.getMimeType(file); - page.setContentType(new Utf8(mime)); + WebPage page = new WebPage(); + page.setBaseUrl(new Utf8(urlString)); + page.setContent(ByteBuffer.wrap(bytes)); + MimeUtil mutil = new MimeUtil(conf); + String mime = mutil.getMimeType(file); + page.setContentType(new Utf8(mime)); - parse = new ParseUtil(conf).parse(urlString, page); - return parse.getOutlinks(); + parse = new ParseUtil(conf).parse(urlString, page); + return parse.getOutlinks(); } @Test public void testOutlinkExtraction() throws ProtocolException, ParseException, IOException { - String[] filenames = new File(sampleDir).list(); - for (int i = 0; i < filenames.length; i++) { - if (filenames[i].endsWith(".js") == true) { - assertEquals("number of outlinks in .js test file should be 5", 5, getOutlinks(sampleFiles)); - // temporarily disabled as a suitable pure JS file could not be be found. - //} else { - //assertEquals("number of outlinks in .html file should be X", 5, getOutlinks(sampleFiles)); - } + String[] filenames = new File(sampleDir).list(); + for (int i = 0; i < filenames.length; i++) { + if (filenames[i].endsWith(".js") == true) { + assertEquals("number of outlinks in .js test file should be 5", 5, getOutlinks(sampleFiles)); + // temporarily disabled as a suitable pure JS file could not be be found. + //} else { + //assertEquals("number of outlinks in .html file should be X", 5, getOutlinks(sampleFiles)); + } } } Index: src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java =================================================================== --- src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java (revision 1484186) +++ src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java (working copy) @@ -21,20 +21,23 @@ import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; /** Unit tests for BasicURLNormalizer. */ -public class TestBasicURLNormalizer extends TestCase { +public class TestBasicURLNormalizer { private BasicURLNormalizer normalizer; private Configuration conf; - public TestBasicURLNormalizer(String name) { - super(name); + @Before + public void setUp() { normalizer = new BasicURLNormalizer(); conf = NutchConfiguration.create(); normalizer.setConf(conf); } + @Test public void testNormalizer() throws Exception { // check that leading and trailing spaces are removed normalizeTest(" http://foo.com/ ", "http://foo.com/"); @@ -109,11 +112,4 @@ assertEquals(normal, normalizer.normalize(weird, URLNormalizers.SCOPE_DEFAULT)); } - public static void main(String[] args) throws Exception { - new TestBasicURLNormalizer("test").testNormalizer(); - } - - - - } \ No newline at end of file Index: src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java =================================================================== --- src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java (revision 1484186) +++ src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java (working copy) @@ -20,13 +20,12 @@ import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; -import org.apache.nutch.fetcher.FetcherJob; import org.apache.nutch.indexer.NutchDocument; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; import org.junit.Test; -import junit.framework.TestCase; +import static org.junit.Assert.*; /** * JUnit test case which tests @@ -39,7 +38,7 @@ * @author lewismc */ -public class TestBasicIndexingFilter extends TestCase { +public class TestBasicIndexingFilter { @Test public void testBasicFields() throws Exception { Index: src/test/org/apache/nutch/fetcher/TestFetcher.java =================================================================== --- src/test/org/apache/nutch/fetcher/TestFetcher.java (revision 1484186) +++ src/test/org/apache/nutch/fetcher/TestFetcher.java (working copy) @@ -30,6 +30,11 @@ import org.apache.nutch.util.CrawlTestUtil; import org.mortbay.jetty.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + /** * Basic fetcher test * 1. generate seedlist @@ -37,7 +42,6 @@ * 3. generate * 3. fetch * 4. Verify contents - * @author nutch-dev * */ public class TestFetcher extends AbstractNutchTest { @@ -47,6 +51,7 @@ Server server; @Override + @Before public void setUp() throws Exception{ super.setUp(); urlPath = new Path(testdir, "urls"); @@ -55,11 +60,13 @@ } @Override + @After public void tearDown() throws Exception{ server.stop(); fs.delete(testdir, true); } + @Test public void testFetch() throws Exception { //generate seedlist @@ -124,6 +131,7 @@ urls.add("http://127.0.0.1:" + server.getConnectors()[0].getPort() + "/" + page); } + @Test public void testAgentNameCheck() { boolean failedNoAgentName = false; Index: src/test/org/apache/nutch/metadata/TestMetadata.java =================================================================== --- src/test/org/apache/nutch/metadata/TestMetadata.java (revision 1484186) +++ src/test/org/apache/nutch/metadata/TestMetadata.java (working copy) @@ -22,30 +22,16 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.Properties; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import junit.textui.TestRunner; +import org.junit.Test; +import static org.junit.Assert.*; /** * JUnit based tests of class {@link org.apache.nutch.metadata.Metadata}. */ -public class TestMetadata extends TestCase { +public class TestMetadata { private static final String CONTENTTYPE = "contenttype"; - public TestMetadata(String testName) { - super(testName); - } - - public static Test suite() { - return new TestSuite(TestMetadata.class); - } - - public static void main(String[] args) { - TestRunner.run(suite()); - } - /** * Test to ensure that only non-null values get written when the * {@link Metadata} object is written using a Writeable. @@ -53,6 +39,7 @@ * @since NUTCH-406 * */ + @Test public void testWriteNonNull() { Metadata met = new Metadata(); met.add(CONTENTTYPE, null); @@ -83,6 +70,7 @@ } /** Test for the add(String, String) method. */ + @Test public void testAdd() { String[] values = null; Metadata meta = new Metadata(); @@ -112,6 +100,7 @@ } /** Test for the set(String, String) method. */ + @Test public void testSet() { String[] values = null; Metadata meta = new Metadata(); @@ -138,6 +127,7 @@ } /** Test for setAll(Properties) method. */ + @Test public void testSetProperties() { String[] values = null; Metadata meta = new Metadata(); @@ -165,6 +155,7 @@ } /** Test for get(String) method. */ + @Test public void testGet() { Metadata meta = new Metadata(); assertNull(meta.get("a-name")); @@ -175,6 +166,7 @@ } /** Test for isMultiValued() method. */ + @Test public void testIsMultiValued() { Metadata meta = new Metadata(); assertFalse(meta.isMultiValued("key")); @@ -185,6 +177,7 @@ } /** Test for names method. */ + @Test public void testNames() { String[] names = null; Metadata meta = new Metadata(); @@ -201,6 +194,7 @@ } /** Test for remove(String) method. */ + @Test public void testRemove() { Metadata meta = new Metadata(); meta.remove("name-one"); @@ -222,6 +216,7 @@ } /** Test for equals(Object) method. */ + @Test public void testObject() { Metadata meta1 = new Metadata(); Metadata meta2 = new Metadata(); @@ -247,6 +242,7 @@ } /** Test for Writable implementation. */ + @Test public void testWritable() { Metadata result = null; Metadata meta = new Metadata(); Index: src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java =================================================================== --- src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java (revision 1484186) +++ src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java (working copy) @@ -22,10 +22,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.Properties; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import junit.textui.TestRunner; +import org.junit.Test; +import static org.junit.Assert.*; /** * JUnit based tests of class @@ -34,23 +32,13 @@ * @author Chris Mattmann * @author Jérôme Charron */ -public class TestSpellCheckedMetadata extends TestCase { +public class TestSpellCheckedMetadata { + private static final int NUM_ITERATIONS = 10000; - public TestSpellCheckedMetadata(String testName) { - super(testName); - } - - public static Test suite() { - return new TestSuite(TestSpellCheckedMetadata.class); - } - - public static void main(String[] args) { - TestRunner.run(suite()); - } - /** Test for the getNormalizedName(String) method. */ + @Test public void testGetNormalizedName() { assertEquals("Content-Type", SpellCheckedMetadata .getNormalizedName("Content-Type")); @@ -67,6 +55,7 @@ } /** Test for the add(String, String) method. */ + @Test public void testAdd() { String[] values = null; SpellCheckedMetadata meta = new SpellCheckedMetadata(); @@ -96,6 +85,7 @@ } /** Test for the set(String, String) method. */ + @Test public void testSet() { String[] values = null; SpellCheckedMetadata meta = new SpellCheckedMetadata(); @@ -122,6 +112,7 @@ } /** Test for setAll(Properties) method. */ + @Test public void testSetProperties() { String[] values = null; SpellCheckedMetadata meta = new SpellCheckedMetadata(); @@ -149,6 +140,7 @@ } /** Test for get(String) method. */ + @Test public void testGet() { SpellCheckedMetadata meta = new SpellCheckedMetadata(); assertNull(meta.get("a-name")); @@ -160,6 +152,7 @@ } /** Test for isMultiValued() method. */ + @Test public void testIsMultiValued() { SpellCheckedMetadata meta = new SpellCheckedMetadata(); assertFalse(meta.isMultiValued("key")); @@ -170,6 +163,7 @@ } /** Test for names method. */ + @Test public void testNames() { String[] names = null; SpellCheckedMetadata meta = new SpellCheckedMetadata(); @@ -186,6 +180,7 @@ } /** Test for remove(String) method. */ + @Test public void testRemove() { SpellCheckedMetadata meta = new SpellCheckedMetadata(); meta.remove("name-one"); @@ -207,6 +202,7 @@ } /** Test for equals(Object) method. */ + @Test public void testObject() { SpellCheckedMetadata meta1 = new SpellCheckedMetadata(); SpellCheckedMetadata meta2 = new SpellCheckedMetadata(); @@ -232,6 +228,7 @@ } /** Test for Writable implementation. */ + @Test public void testWritable() { SpellCheckedMetadata result = null; SpellCheckedMetadata meta = new SpellCheckedMetadata(); @@ -259,6 +256,7 @@ * IO Test method, usable only when you plan to do changes in metadata * to measure relative performance impact. */ + @Test public final void testHandlingSpeed() { SpellCheckedMetadata result; long start = System.currentTimeMillis(); Index: src/test/org/apache/nutch/protocol/TestProtocolFactory.java =================================================================== --- src/test/org/apache/nutch/protocol/TestProtocolFactory.java (revision 1484186) +++ src/test/org/apache/nutch/protocol/TestProtocolFactory.java (working copy) @@ -20,20 +20,24 @@ import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.ObjectCache; -import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestProtocolFactory extends TestCase { +public class TestProtocolFactory { Configuration conf; ProtocolFactory factory; - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { conf = NutchConfiguration.create(); conf.set("plugin.includes", ".*"); conf.set("http.agent.name", "test-bot"); factory=new ProtocolFactory(conf); } + @Test public void testGetProtocol(){ //non existing protocol @@ -69,6 +73,7 @@ } } + @Test public void testContains(){ assertTrue(factory.contains("http", "http")); assertTrue(factory.contains("http", "http,ftp")); Index: src/test/org/apache/nutch/protocol/TestContent.java =================================================================== --- src/test/org/apache/nutch/protocol/TestContent.java (revision 1484186) +++ src/test/org/apache/nutch/protocol/TestContent.java (working copy) @@ -24,17 +24,17 @@ import org.apache.nutch.util.WritableTestUtils; import org.apache.tika.mime.MimeTypes; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** Unit tests for Content. */ -public class TestContent extends TestCase { +public class TestContent { private static Configuration conf = NutchConfiguration.create(); - public TestContent(String name) { super(name); } - + @Test public void testContent() throws Exception { String page = "

Hello World

The Quick Brown Fox Jumped Over the Lazy Fox."; @@ -55,6 +55,7 @@ } /** Unit tests for getContentType(String, String, byte[]) method. */ + @Test public void testGetContentType() throws Exception { Content c = null; Metadata p = new Metadata(); Index: src/test/org/apache/nutch/storage/TestGoraStorage.java =================================================================== --- src/test/org/apache/nutch/storage/TestGoraStorage.java (revision 1484186) +++ src/test/org/apache/nutch/storage/TestGoraStorage.java (working copy) @@ -35,16 +35,34 @@ import org.apache.nutch.util.CrawlTestUtil; import org.hsqldb.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + /** * Tests basic Gora functionality by writing and reading webpages. */ public class TestGoraStorage extends AbstractNutchTest { + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + } + /** * Sequentially read and write pages to a store. * * @throws Exception */ + @Test public void testSinglethreaded() throws Exception { String id = "singlethread"; readWrite(id, webPageStore); @@ -87,6 +105,7 @@ * * @throws Exception */ + @Test public void testMultithreaded() throws Exception { // create a fixed thread pool int numThreads = 8; @@ -127,6 +146,7 @@ * * @throws Exception */ + @Test public void testMultiProcess() throws Exception { // create and start a hsql server, a stand-alone (memory backed) db // (important: a stand-alone server should be used because simple Index: src/test/org/apache/nutch/net/TestURLNormalizers.java =================================================================== --- src/test/org/apache/nutch/net/TestURLNormalizers.java (revision 1484186) +++ src/test/org/apache/nutch/net/TestURLNormalizers.java (working copy) @@ -21,10 +21,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestURLNormalizers extends TestCase { +public class TestURLNormalizers { + @Test public void testURLNormalizers() { Configuration conf = NutchConfiguration.create(); String clazz1 = "org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer"; Index: src/test/org/apache/nutch/net/TestURLFilters.java =================================================================== --- src/test/org/apache/nutch/net/TestURLFilters.java (revision 1484186) +++ src/test/org/apache/nutch/net/TestURLFilters.java (working copy) @@ -19,14 +19,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestURLFilters extends TestCase { +public class TestURLFilters { /** * Testcase for NUTCH-325. * @throws URLFilterException */ + @Test public void testNonExistingUrlFilter() throws URLFilterException { Configuration conf = NutchConfiguration.create(); String class1 = "NonExistingFilter"; @@ -34,7 +36,12 @@ conf.set(URLFilters.URLFILTER_ORDER, class1 + " " + class2); URLFilters normalizers = new URLFilters(conf); - normalizers.filter("http://someurl/"); + assertNotNull(normalizers); + try { + normalizers.filter("http://someurl/"); + } catch (URLFilterException ufe) { + fail(ufe.toString()); + } } } Index: src/test/org/apache/nutch/crawl/TestSignatureFactory.java =================================================================== --- src/test/org/apache/nutch/crawl/TestSignatureFactory.java (revision 1484186) +++ src/test/org/apache/nutch/crawl/TestSignatureFactory.java (working copy) @@ -19,10 +19,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestSignatureFactory extends TestCase { +public class TestSignatureFactory { + @Test public void testGetSignature() { Configuration conf=NutchConfiguration.create(); Signature signature1=SignatureFactory.getSignature(conf); Index: src/test/org/apache/nutch/crawl/TestUrlWithScore.java =================================================================== --- src/test/org/apache/nutch/crawl/TestUrlWithScore.java (revision 1484186) +++ src/test/org/apache/nutch/crawl/TestUrlWithScore.java (working copy) @@ -22,7 +22,8 @@ import java.io.DataOutputStream; import java.io.IOException; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.hadoop.io.RawComparator; import org.apache.nutch.crawl.UrlWithScore; @@ -33,8 +34,9 @@ /** * Tests {@link UrlWithScore} with serialization, partitioning and sorting. */ -public class TestUrlWithScore extends TestCase { +public class TestUrlWithScore { + @Test public void testSerialization() throws IOException { // create a key and test basic functionality UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f); @@ -58,6 +60,7 @@ out.close(); } + @Test public void testPartitioner() throws IOException { UrlOnlyPartitioner part = new UrlOnlyPartitioner(); @@ -80,6 +83,7 @@ assertEquals(partForKey3, part.getPartition(k5, null, numReduces)); } + @Test public void testUrlOnlySorting() throws IOException { UrlOnlyComparator comp = new UrlOnlyComparator(); @@ -104,6 +108,7 @@ assertEquals(1, compareBothRegularAndRaw(comp, k5, k1)); } + @Test public void testUrlScoreSorting() throws IOException { UrlScoreComparator comp = new UrlScoreComparator(); Index: src/test/org/apache/nutch/crawl/TestInjector.java =================================================================== --- src/test/org/apache/nutch/crawl/TestInjector.java (revision 1484186) +++ src/test/org/apache/nutch/crawl/TestInjector.java (working copy) @@ -28,6 +28,8 @@ import org.apache.nutch.util.CrawlTestUtil; import org.apache.gora.util.ByteUtils; import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; /** * Basic injector test: 1. Creates a text file with urls 2. Injects them into @@ -46,6 +48,7 @@ urlPath = new Path(testdir, "urls"); } + @Test public void testInject() throws Exception { ArrayList urls = new ArrayList(); for (int i = 0; i < 100; i++) { Index: src/test/org/apache/nutch/crawl/TestURLPartitioner.java =================================================================== --- src/test/org/apache/nutch/crawl/TestURLPartitioner.java (revision 1484186) +++ src/test/org/apache/nutch/crawl/TestURLPartitioner.java (working copy) @@ -18,7 +18,8 @@ import java.net.MalformedURLException; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IntWritable; @@ -33,11 +34,12 @@ /** * Tests {@link URLPartitioner} */ -public class TestURLPartitioner extends TestCase { +public class TestURLPartitioner { /** * tests one reducer, everything goes into one partition, using host partitioner. */ + @Test public void testOneReducer() { URLPartitioner partitioner = new URLPartitioner(); Configuration conf = NutchConfiguration.create(); @@ -53,6 +55,7 @@ /** * tests partitioning by host */ + @Test public void testModeHost() { URLPartitioner partitioner = new URLPartitioner(); Configuration conf = NutchConfiguration.create(); @@ -75,6 +78,7 @@ /** * tests partitioning by domain */ + @Test public void testModeDomain() { URLPartitioner partitioner = new URLPartitioner(); Configuration conf = NutchConfiguration.create(); @@ -97,6 +101,7 @@ /** * tests partitioning by IP */ + @Test public void testModeIP() { URLPartitioner partitioner = new URLPartitioner(); Configuration conf = NutchConfiguration.create(); @@ -117,10 +122,10 @@ partitionWithoutWWW, partitionWithWWW); } - /** * Test the seed functionality, using host partitioner. */ + @Test public void testSeed() { URLPartitioner partitioner = new URLPartitioner(); Configuration conf = NutchConfiguration.create(); @@ -143,6 +148,7 @@ /** * Tests the {@link SelectorEntryPartitioner}. */ + @Test public void testSelectorEntryPartitioner() { //The reference partitioner URLPartitioner refPartitioner = new URLPartitioner(); @@ -174,6 +180,7 @@ * Tests the {@link FetchEntryPartitioner} * @throws MalformedURLException */ + @Test public void testFetchEntryPartitioner() throws MalformedURLException { //The reference partitioner URLPartitioner refPartitioner = new URLPartitioner(); Index: src/test/org/apache/nutch/crawl/TestGenerator.java =================================================================== --- src/test/org/apache/nutch/crawl/TestGenerator.java (revision 1484186) +++ src/test/org/apache/nutch/crawl/TestGenerator.java (working copy) @@ -28,15 +28,16 @@ import org.apache.nutch.util.AbstractNutchTest; import org.apache.nutch.util.CrawlTestUtil; import org.apache.nutch.util.TableUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; /** * Basic generator test. 1. Insert entries in webtable 2. Generates entries to * fetch 3. Verifies that number of generated urls match 4. Verifies that * highest scoring urls are generated * - * @author nutch-dev - * @param - * */ public class TestGenerator extends AbstractNutchTest { @@ -46,12 +47,25 @@ WebPage.Field.MARKERS.getName(), WebPage.Field.SCORE.getName() }; + + @Override + @Before + public void setUp() throws Exception{ + super.setUp(); + } + + @Override + @After + public void tearDown()throws Exception { + super.tearDown(); + } /** * Test that generator generates fetchlist ordered by score (desc). * * @throws Exception */ + @Test public void testGenerateHighest() throws Exception { final int NUM_RESULTS = 2; @@ -111,6 +125,7 @@ * * @throws Exception */ + @Test public void testGenerateHostLimit() throws Exception { ArrayList list = new ArrayList(); @@ -158,6 +173,7 @@ * * @throws Exception */ + @Test public void testGenerateDomainLimit() throws Exception { ArrayList list = new ArrayList(); @@ -209,6 +225,7 @@ * @throws Exception * @throws IOException */ + @Test public void testFilter() throws IOException, Exception { ArrayList list = new ArrayList(); Index: src/test/org/apache/nutch/parse/TestOutlinkExtractor.java =================================================================== --- src/test/org/apache/nutch/parse/TestOutlinkExtractor.java (revision 1484186) +++ src/test/org/apache/nutch/parse/TestOutlinkExtractor.java (working copy) @@ -22,7 +22,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.NutchConfiguration; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** * TestCase to check regExp extraction of URLs. @@ -31,7 +32,7 @@ * * @version 1.0 */ -public class TestOutlinkExtractor extends TestCase { +public class TestOutlinkExtractor { private static Configuration conf = NutchConfiguration.create(); public void testGetNoOutlinks() { @@ -46,6 +47,7 @@ assertEquals(0, outlinks.length); } + @Test public void testGetOutlinksHttp() { Outlink[] outlinks = OutlinkExtractor.getOutlinks( "Test with http://www.nutch.org/index.html is it found? " + @@ -58,6 +60,7 @@ assertEquals("Wrong URL", "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl()); } + @Test public void testGetOutlinksHttp2() { Outlink[] outlinks = OutlinkExtractor.getOutlinks( "Test with http://www.nutch.org/index.html is it found? " + @@ -69,6 +72,8 @@ assertEquals("Wrong URL", "http://www.google.de", outlinks[1].getToUrl()); assertEquals("Wrong URL", "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl()); } + + @Test public void testGetOutlinksFtp() { Outlink[] outlinks = OutlinkExtractor.getOutlinks( "Test with ftp://www.nutch.org is it found? " + Index: src/test/org/apache/nutch/parse/TestParserFactory.java =================================================================== --- src/test/org/apache/nutch/parse/TestParserFactory.java (revision 1484186) +++ src/test/org/apache/nutch/parse/TestParserFactory.java (working copy) @@ -17,8 +17,9 @@ package org.apache.nutch.parse; -// JUnit imports -import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; // Nutch imports import org.apache.nutch.plugin.Extension; @@ -29,17 +30,15 @@ * Unit test for new parse plugin selection. * * @author Sebastien Le Callonnec - * @version 1.0 */ -public class TestParserFactory extends TestCase { +public class TestParserFactory { private Configuration conf; private ParserFactory parserFactory; - public TestParserFactory(String name) { super(name); } - /** Inits the Test Case with the test parse-plugin file */ - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { conf = NutchConfiguration.create(); conf.set("plugin.includes", ".*"); conf.set("parse.plugin.file", @@ -48,6 +47,7 @@ } /** Unit test for getExtensions(String) method. */ + @Test public void testGetExtensions() throws Exception { Extension ext = parserFactory.getExtensions("text/html").get(0); assertEquals("parse-tika", ext.getDescriptor().getPluginId()); @@ -58,6 +58,7 @@ } /** Unit test to check getParsers method */ + @Test public void testGetParsers() throws Exception { Parser [] parsers = parserFactory.getParsers("text/html", "http://foo.com"); assertNotNull(parsers); Index: src/test/org/apache/nutch/util/TestStringUtil.java =================================================================== --- src/test/org/apache/nutch/util/TestStringUtil.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestStringUtil.java (working copy) @@ -17,14 +17,13 @@ package org.apache.nutch.util; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** Unit tests for StringUtil methods. */ -public class TestStringUtil extends TestCase { - public TestStringUtil(String name) { - super(name); - } +public class TestStringUtil { + @Test public void testRightPad() { String s= "my string"; @@ -42,6 +41,7 @@ } + @Test public void testLeftPad() { String s= "my string"; Index: src/test/org/apache/nutch/util/AbstractNutchTest.java =================================================================== --- src/test/org/apache/nutch/util/AbstractNutchTest.java (revision 1484186) +++ src/test/org/apache/nutch/util/AbstractNutchTest.java (working copy) @@ -16,8 +16,6 @@ */ package org.apache.nutch.util; -import junit.framework.TestCase; - import org.apache.gora.store.DataStore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -29,7 +27,7 @@ * This class provides common routines for setup/teardown of an in-memory data * store. */ -public class AbstractNutchTest extends TestCase { +public class AbstractNutchTest { protected Configuration conf; protected FileSystem fs; @@ -37,9 +35,7 @@ protected DataStore webPageStore; protected boolean persistentDataStore = false; - @Override public void setUp() throws Exception { - super.setUp(); conf = CrawlTestUtil.createConfiguration(); conf.set("storage.data.store.class", "org.apache.gora.sql.store.SqlStore"); fs = FileSystem.get(conf); @@ -47,7 +43,6 @@ WebPage.class); } - @Override public void tearDown() throws Exception { // empty the database after test if (!persistentDataStore) { @@ -55,7 +50,6 @@ webPageStore.flush(); webPageStore.close(); } - super.tearDown(); fs.delete(testdir, true); } Index: src/test/org/apache/nutch/util/TestTableUtil.java =================================================================== --- src/test/org/apache/nutch/util/TestTableUtil.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestTableUtil.java (working copy) @@ -17,9 +17,10 @@ package org.apache.nutch.util; import org.apache.nutch.util.TableUtil; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; -public class TestTableUtil extends TestCase { +public class TestTableUtil { String urlString1 = "http://foo.com/"; String urlString2 = "http://foo.com:8900/"; @@ -38,6 +39,7 @@ String reversedUrlString6 = "com.foo:http"; String reversedUrlString7 = ":file/var/www/index.html"; + @Test public void testReverseUrl() throws Exception { assertReverse(urlString1, reversedUrlString1); assertReverse(urlString2, reversedUrlString2); @@ -49,6 +51,7 @@ assertReverse(urlString7, reversedUrlString7); } + @Test public void testUnreverseUrl() throws Exception { assertUnreverse(reversedUrlString1, urlString1); assertUnreverse(reversedUrlString2, urlString2); Index: src/test/org/apache/nutch/util/TestPrefixStringMatcher.java =================================================================== --- src/test/org/apache/nutch/util/TestPrefixStringMatcher.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestPrefixStringMatcher.java (working copy) @@ -17,13 +17,11 @@ package org.apache.nutch.util; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** Unit tests for PrefixStringMatcher. */ -public class TestPrefixStringMatcher extends TestCase { - public TestPrefixStringMatcher(String name) { - super(name); - } +public class TestPrefixStringMatcher { private final static int NUM_TEST_ROUNDS= 20; private final static int MAX_TEST_PREFIXES= 100; @@ -51,6 +49,7 @@ return new String(chars); } + @Test public void testPrefixMatcher() { int numMatches= 0; int numInputsTested= 0; Index: src/test/org/apache/nutch/util/TestGZIPUtils.java =================================================================== --- src/test/org/apache/nutch/util/TestGZIPUtils.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestGZIPUtils.java (working copy) @@ -17,15 +17,13 @@ package org.apache.nutch.util; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import java.io.IOException; /** Unit tests for GZIPUtils methods. */ -public class TestGZIPUtils extends TestCase { - public TestGZIPUtils(String name) { - super(name); - } +public class TestGZIPUtils { /* a short, highly compressable, string */ String SHORT_TEST_STRING= @@ -117,6 +115,7 @@ // tests + @Test public void testZipUnzip() { byte[] testBytes= SHORT_TEST_STRING.getBytes(); testZipUnzip(testBytes); @@ -126,6 +125,7 @@ testZipUnzip(testBytes); } + @Test public void testZipUnzipBestEffort() { byte[] testBytes= SHORT_TEST_STRING.getBytes(); testZipUnzipBestEffort(testBytes); @@ -135,6 +135,7 @@ testZipUnzipBestEffort(testBytes); } + @Test public void testTruncation() { byte[] testBytes= SHORT_TEST_STRING.getBytes(); testTruncation(testBytes); @@ -144,6 +145,7 @@ testTruncation(testBytes); } + @Test public void testLimit() { byte[] testBytes= SHORT_TEST_STRING.getBytes(); testLimit(testBytes); Index: src/test/org/apache/nutch/util/TestNodeWalker.java =================================================================== --- src/test/org/apache/nutch/util/TestNodeWalker.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestNodeWalker.java (working copy) @@ -18,20 +18,17 @@ package org.apache.nutch.util; import java.io.ByteArrayInputStream; -import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + import org.apache.xerces.parsers.DOMParser; import org.w3c.dom.Node; import org.xml.sax.InputSource; - - - /** Unit tests for NodeWalker methods. */ -public class TestNodeWalker extends TestCase { - public TestNodeWalker(String name) { - super(name); - } +public class TestNodeWalker { /* a snapshot of the nutch webpage */ private final static String WEBPAGE= @@ -49,13 +46,15 @@ private final static String[] ULCONTENT = new String[4]; - protected void setUp() throws Exception{ + @Before + public void setUp() throws Exception{ ULCONTENT[0]="crawl several billion pages per month" ; ULCONTENT[1]="maintain an index of these pages" ; ULCONTENT[2]="search that index up to 1000 times per second" ; ULCONTENT[3]="operate at minimal cost" ; } + @Test public void testSkipChildren() { DOMParser parser= new DOMParser(); Index: src/test/org/apache/nutch/util/TestSuffixStringMatcher.java =================================================================== --- src/test/org/apache/nutch/util/TestSuffixStringMatcher.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestSuffixStringMatcher.java (working copy) @@ -17,13 +17,11 @@ package org.apache.nutch.util; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** Unit tests for SuffixStringMatcher. */ -public class TestSuffixStringMatcher extends TestCase { - public TestSuffixStringMatcher(String name) { - super(name); - } +public class TestSuffixStringMatcher { private final static int NUM_TEST_ROUNDS= 20; private final static int MAX_TEST_SUFFIXES= 100; @@ -51,6 +49,7 @@ return new String(chars); } + @Test public void testSuffixMatcher() { int numMatches= 0; int numInputsTested= 0; Index: src/test/org/apache/nutch/util/TestEncodingDetector.java =================================================================== --- src/test/org/apache/nutch/util/TestEncodingDetector.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestEncodingDetector.java (working copy) @@ -19,14 +19,15 @@ import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.protocols.Response; import org.apache.nutch.storage.WebPage; -public class TestEncodingDetector extends TestCase { +public class TestEncodingDetector { private static Configuration conf = NutchConfiguration.create(); private static byte[] contentInOctets; @@ -39,10 +40,7 @@ } } - public TestEncodingDetector(String name) { - super(name); - } - + @Test public void testGuessing() { // first disable auto detection conf.setInt(EncodingDetector.MIN_CONFIDENCE_KEY, -1); Index: src/test/org/apache/nutch/util/TestURLUtil.java =================================================================== --- src/test/org/apache/nutch/util/TestURLUtil.java (revision 1484186) +++ src/test/org/apache/nutch/util/TestURLUtil.java (working copy) @@ -19,18 +19,13 @@ import java.net.URL; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; /** Test class for URLUtil */ -public class TestURLUtil - extends TestCase { +public class TestURLUtil { - @Override - protected void setUp() - throws Exception { - super.setUp(); - } - + @Test public void testGetDomainName() throws Exception { @@ -81,6 +76,7 @@ } + @Test public void testGetDomainSuffix() throws Exception { URL url = null; @@ -133,6 +129,7 @@ } + @Test public void testGetHostBatches() throws Exception { URL url; @@ -165,6 +162,7 @@ } + @Test public void testChooseRepr() throws Exception { Index: src/test/org/apache/nutch/indexer/TestIndexingFilters.java =================================================================== --- src/test/org/apache/nutch/indexer/TestIndexingFilters.java (revision 1484186) +++ src/test/org/apache/nutch/indexer/TestIndexingFilters.java (working copy) @@ -16,7 +16,8 @@ */ package org.apache.nutch.indexer; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; @@ -24,12 +25,13 @@ import org.apache.nutch.storage.WebPage; import org.apache.nutch.util.NutchConfiguration; -public class TestIndexingFilters extends TestCase { +public class TestIndexingFilters { /** * Test behaviour when defined filter does not exist. * @throws IndexingException */ + @Test public void testNonExistingIndexingFilter() throws IndexingException { Configuration conf = NutchConfiguration.create(); conf.addResource("nutch-default.xml"); @@ -50,6 +52,7 @@ * Test behaviour when NutchDOcument is null * @throws IndexingException */ + @Test public void testNutchDocumentNullIndexingFilter() throws IndexingException{ Configuration conf = NutchConfiguration.create(); conf.addResource("nutch-default.xml"); @@ -64,38 +67,38 @@ assertNull(doc); } - /** - * Test behaviour when reset the index filter order will not take effect - * - * @throws IndexingException - */ - public void testFilterCacheIndexingFilter() throws IndexingException{ - Configuration conf = NutchConfiguration.create(); - conf.addResource("nutch-default.xml"); - conf.addResource("crawl-tests.xml"); + /** + * Test behaviour when reset the index filter order will not take effect + * + * @throws IndexingException + */ + @Test + public void testFilterCacheIndexingFilter() throws IndexingException{ + Configuration conf = NutchConfiguration.create(); + conf.addResource("nutch-default.xml"); + conf.addResource("crawl-tests.xml"); - String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter"; - conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1); + String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter"; + conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1); - IndexingFilters filters1 = new IndexingFilters(conf); - WebPage page = new WebPage(); - page.setText(new Utf8("text")); - page.setTitle(new Utf8("title")); - NutchDocument fdoc1 = filters1.filter(new NutchDocument(),"http://www.example.com/",page); + IndexingFilters filters1 = new IndexingFilters(conf); + WebPage page = new WebPage(); + page.setText(new Utf8("text")); + page.setTitle(new Utf8("title")); + NutchDocument fdoc1 = filters1.filter(new NutchDocument(),"http://www.example.com/",page); - // add another index filter - String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer"; - // set content metadata - Metadata md = new Metadata(); - md.add("example","data"); - // set content metadata property defined in MetadataIndexer - conf.set("index.content.md","example"); - // add MetadataIndxer filter - conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2); - IndexingFilters filters2 = new IndexingFilters(conf); - NutchDocument fdoc2 = filters2.filter(new NutchDocument(),"http://www.example.com/",page); - assertEquals(fdoc1.getFieldNames().size(),fdoc2.getFieldNames().size()); - } + // add another index filter + String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer"; + // set content metadata + Metadata md = new Metadata(); + md.add("example","data"); + // set content metadata property defined in MetadataIndexer + conf.set("index.content.md","example"); + // add MetadataIndxer filter + conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2); + IndexingFilters filters2 = new IndexingFilters(conf); + NutchDocument fdoc2 = filters2.filter(new NutchDocument(),"http://www.example.com/",page); + assertEquals(fdoc1.getFieldNames().size(),fdoc2.getFieldNames().size()); + } - } Index: src/test/org/apache/nutch/plugin/TestPluginSystem.java =================================================================== --- src/test/org/apache/nutch/plugin/TestPluginSystem.java (revision 1484186) +++ src/test/org/apache/nutch/plugin/TestPluginSystem.java (working copy) @@ -26,12 +26,14 @@ import java.util.Locale; import java.util.Properties; -import junit.framework.TestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.nutch.util.NutchConfiguration; -import org.apache.nutch.util.NutchJob; import org.apache.nutch.util.NutchJobConf; /** @@ -39,14 +41,15 @@ * * @author joa23 */ -public class TestPluginSystem extends TestCase { +public class TestPluginSystem { private int fPluginCount; - private LinkedList fFolders = new LinkedList(); + private LinkedList fFolders = new LinkedList(); private Configuration conf ; private PluginRepository repository; - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { this.conf = NutchConfiguration.create(); conf.set("plugin.includes", ".*"); // String string = this.conf.get("plugin.includes", ""); @@ -56,12 +59,8 @@ this.repository = PluginRepository.get(conf); } - /* - * (non-Javadoc) - * - * @see junit.framework.TestCase#tearDown() - */ - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { for (int i = 0; i < fFolders.size(); i++) { File folder = (File) fFolders.get(i); delete(folder); @@ -72,6 +71,7 @@ /** */ + @Test public void testPluginConfiguration() { String string = getPluginFolder(); File file = new File(string); @@ -83,6 +83,7 @@ /** */ + @Test public void testLoadPlugins() { PluginDescriptor[] descriptors = repository .getPluginDescriptors(); @@ -98,6 +99,7 @@ } } + @Test public void testRepositoryCache() { Configuration config = NutchConfiguration.create(); PluginRepository repo = PluginRepository.get(config); @@ -117,6 +119,7 @@ /** * */ + @Test public void testGetExtensionAndAttributes() { String xpId = " sdsdsd"; ExtensionPoint extensionPoint =repository @@ -135,6 +138,7 @@ /** * @throws PluginRuntimeException */ + @Test public void testGetExtensionInstances() throws PluginRuntimeException { Extension[] extensions = repository .getExtensionPoint(getGetExtensionId()).getExtensions(); @@ -154,6 +158,7 @@ * * */ + @Test public void testGetClassLoader() { PluginDescriptor[] descriptors = repository .getPluginDescriptors(); @@ -166,6 +171,7 @@ /** * @throws IOException */ + @Test public void testGetResources() throws IOException { PluginDescriptor[] descriptors = repository .getPluginDescriptors();