diff --git src/java/org/apache/nutch/crawl/DbUpdateMapper.java src/java/org/apache/nutch/crawl/DbUpdateMapper.java
index 5bef21d..8b25eb5 100644
--- src/java/org/apache/nutch/crawl/DbUpdateMapper.java
+++ src/java/org/apache/nutch/crawl/DbUpdateMapper.java
@@ -17,6 +17,7 @@
 package org.apache.nutch.crawl;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -27,7 +28,6 @@ import org.apache.nutch.metadata.Nutch;
 import org.apache.nutch.storage.Mark;
 import org.apache.nutch.util.NutchJob;
 import org.slf4j.Logger;
-import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.nutch.scoring.ScoreDatum;
 import org.apache.nutch.scoring.ScoringFilterException;
@@ -51,6 +51,9 @@ extends GoraMapper<String, WebPage, UrlWithScore, NutchWritable> {
   private UrlWithScore urlWithScore = new UrlWithScore();
   private NutchWritable nutchWritable = new NutchWritable();
   private WebPageWritable pageWritable;
+  
+  private WebPage webpageCanonical;
+  private static final Utf8 YES_STRING = new Utf8("y");
 
   @Override
   public void map(String key, WebPage page, Context context)
@@ -100,6 +103,17 @@ extends GoraMapper<String, WebPage, UrlWithScore, NutchWritable> {
       nutchWritable.set(scoreDatum);
       context.write(urlWithScore, nutchWritable);
     }
+    webpageCanonical = new WebPage();
+    ByteBuffer getCanonical = page.getFromMetadata(new Utf8("canonical"));
+    if (getCanonical != null) {
+      urlWithScore
+          .setUrl(TableUtil.reverseUrl(new String(getCanonical.array())));
+      urlWithScore.setScore(Float.MAX_VALUE);
+      pageWritable.setWebPage(webpageCanonical);
+      nutchWritable.set(pageWritable);
+      Mark.INJECT_MARK.putMark(webpageCanonical, YES_STRING);
+      context.write(urlWithScore, nutchWritable);
+    }
   }
 
   @Override
diff --git src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMContentUtils.java src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMContentUtils.java
index 56ae989..377245c 100644
--- src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMContentUtils.java
+++ src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMContentUtils.java
@@ -26,7 +26,8 @@ import java.util.HashMap;
 import org.apache.nutch.parse.Outlink;
 import org.apache.nutch.util.NodeWalker;
 import org.apache.hadoop.conf.Configuration;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.w3c.dom.*;
 
 /**
@@ -37,6 +38,8 @@ import org.w3c.dom.*;
  *
  */
 public class DOMContentUtils {
+  public static final Logger LOG = LoggerFactory
+      .getLogger("org.apache.nutch.parse.html");
 
   public static class LinkParams {
     public String elName;
@@ -411,6 +414,45 @@ public class DOMContentUtils {
       }
     }
   }
+  // Better to pass urls throught regexp url normalizer before comparing?
+  private String normalizeUrl(String url) {
+    return url.replaceFirst("/$", "");
+  }
+
+  /**
+   * Check if page supplied in DOM <code>node</code> has canonical link element,
+   * if so compare it to page <code>url</code> if those are different then page
+   * is not canonical.
+   */
+  public URL getDifferentCanonical(Node node, URL url) {
+    NodeWalker walker = new NodeWalker(node);
+    while (walker.hasNext()) {
+
+      Node currentNode = walker.nextNode();
+      String nodeName = currentNode.getNodeName();
+      if ("link".equalsIgnoreCase(nodeName)) {
+        NamedNodeMap attrs = currentNode.getAttributes();
+        Node rel = attrs.getNamedItem("rel");
+        if (rel != null && "canonical".equalsIgnoreCase(rel.getNodeValue())) {
+          String href = attrs.getNamedItem("href").getNodeValue();
+          try {
+            URL canonical = new URL(href);
+            if (!checkUrlEquality(canonical, url)) {
+              return canonical;
+            }
+          } catch (MalformedURLException e) {
+            LOG.warn("There is an error.When check canonical and base url equaltiy.");
+          }
+        }
+      }
+    }
+    return null;
+  }
 
+  public boolean checkUrlEquality(URL canonical, URL url) {
+    return normalizeUrl(canonical.getFile()).equalsIgnoreCase(
+        normalizeUrl(url.getFile()))
+        && canonical.getHost().equalsIgnoreCase(url.getHost());
+  }
 }
 
diff --git src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
index 0a7226f..e3b5ee5 100644
--- src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
+++ src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
@@ -67,6 +67,7 @@ public class HtmlParser implements Parser {
   // meta tag well past the first 1000 bytes.
   // (e.g. http://cn.promo.yahoo.com/customcare/music.html)
   private static final int CHUNK_SIZE = 2000;
+  private static final String CANONICAL_VALUE = "canonical";
 
   // NUTCH-1006 Meta equiv with single quotes not accepted
   private static Pattern metaPattern =
@@ -184,7 +185,12 @@ public class HtmlParser implements Parser {
       LOG.error("Failed with the following Exception: ", e);
       return ParseStatusUtils.getEmptyParse(e, getConf());
     }
-
+    // get and control canonical(rel="canonical") value.
+    URL canonical = utils.getDifferentCanonical(root, base);
+    if (canonical != null) {
+      LOG.warn("Non canonical page. Ignore: " + base);
+      return signCanonical(canonical, page);
+    }
     // get meta directives
     HTMLMetaProcessor.getMetaTags(metaTags, root, base);
     if (LOG.isTraceEnabled()) {
@@ -231,7 +237,13 @@ public class HtmlParser implements Parser {
 
     return parse;
   }
-
+  // write canonical url to metadata and return Empty Parse.
+  public Parse signCanonical(URL base, WebPage wp) {
+    wp.putToMetadata(new Utf8(CANONICAL_VALUE),
+        ByteBuffer.wrap(base.toString().getBytes()));
+    return ParseStatusUtils.getEmptyParse(999, "Non canonical page. Ignore",
+        getConf());
+  }  
   private DocumentFragment parse(InputSource input) throws Exception {
     if (parserImpl.equalsIgnoreCase("tagsoup"))
       return parseTagSoup(input);
diff --git src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java
index 4edfa37..1f53ab8 100644
--- src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java
+++ src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java
@@ -18,28 +18,38 @@
 package org.apache.nutch.parse.html;
 
 import org.apache.nutch.parse.Outlink;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseStatusCodes;
+import org.apache.avro.util.Utf8;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.Bytes;
 import org.apache.nutch.util.NutchConfiguration;
 
 import java.io.ByteArrayInputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.StringTokenizer;
 
 import org.cyberneko.html.parsers.*;
 import org.xml.sax.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.w3c.dom.*;
 import org.apache.html.dom.*;
-
 import org.junit.Before;
 import org.junit.Test;
+
 import static org.junit.Assert.*;
 
 /** 
  * Unit tests for DOMContentUtils.
  */
 public class TestDOMContentUtils {
+  public static final Logger LOG = LoggerFactory
+      .getLogger("org.apache.nutch.parse.html");
 
   private static final String[] testPages= { 
     new String("<html><head><title> title </title><script> script </script>"
@@ -136,11 +146,13 @@ public class TestDOMContentUtils {
             + "<input type=submit><p>test1</p></form>"
             + "<form method='GET' action='/dummy.jsp'><input type=text>"
             + "<input type=submit><p>test2</p></form></body></html>"),
+    // Also test canonical
     new String("<html><head><title> title </title>"
       + "</head><body>"
       + "<a href=\";x\">anchor1</a>"
       + "<a href=\"g;x\">anchor2</a>"
       + "<a href=\"g;x?y#s\">anchor3</a>"
+      + "<link rel=\"canonical\" href=\"http://www.canonical.com/\">canonical</a>"
       + "</body></html>"),  
     new String("<html><head><title> title </title>"
         + "</head><body>"
@@ -165,6 +177,7 @@ public class TestDOMContentUtils {
     "http://www.nutch.org//",
     "http://www.nutch.org/",
     "http://www.nutch.org/",
+     // Also baseurl to test canonical
     "http://www.nutch.org/",
     "http://www.nutch.org/;something"
   };
@@ -192,7 +205,7 @@ public class TestDOMContentUtils {
     "ignore ignore",
     "test1 test2",
     "test1 test2",
-    "title anchor1 anchor2 anchor3",
+    "title anchor1 anchor2 anchor3 canonical",
     "title anchor1 anchor2 anchor3 anchor4 anchor5"
   };
 
@@ -215,6 +228,8 @@ public class TestDOMContentUtils {
   private static Outlink[][] answerOutlinks;
   
   private static Configuration conf;
+  
+  private String canonicalUrl = "http://www.canonical.com/";
   private static DOMContentUtils utils = null;
   
   @Before
@@ -286,7 +301,9 @@ public class TestDOMContentUtils {
         {
           new Outlink("http://www.nutch.org/;x", "anchor1"),
           new Outlink("http://www.nutch.org/g;x", "anchor2"),
-          new Outlink("http://www.nutch.org/g;x?y#s", "anchor3")
+          new Outlink("http://www.nutch.org/g;x?y#s", "anchor3"),
+          // canonical
+          new Outlink("http://www.canonical.com/", ""),
         },
         {
           new Outlink("http://www.nutch.org/g;something", "anchor1"),
@@ -368,6 +385,81 @@ public class TestDOMContentUtils {
       compareOutlinks(answerOutlinks[i], outlinkArr);
     }
   }
+  
+  /**
+   * 
+   * TestCase to test Canonical situations.
+   */
+  @Test
+  public void testCanonical() {
+    if (testDOMs[0] == null)
+      setup();
+    for (int i = 0; i < testPages.length; i++) {
+      if (i == 10) {
+        try {
+          compareCanonicalLink(testDOMs[i], new URL(testBaseHrefs[i]));
+          compareCanonicalParse(testBaseHrefs[i], testPages[i]);
+        } catch (MalformedURLException e) {
+          LOG.warn("There is an error.When test canonical link or Parsed canonical page.");
+        }
+      } else
+        continue;
+    }
+  }
+
+  /**
+   * 
+   * If url of page which has canonical tag not equals href attribute of
+   * value.And canonical enables via confugartion. Should return empty Parse.
+   * Method compares conclusions of Parse.
+   * 
+   * @param baseUrl
+   * @param content
+   */
+  public void compareCanonicalParse(String baseUrl, String content) {
+
+    WebPage wp = prepareWebpage(baseUrl, content);
+    HtmlParser hp = new HtmlParser();
+    hp.setConf(conf);
+    Parse parse = hp.getParse(baseUrl, wp);
+    ByteBuffer urlFromMetadata = wp.getFromMetadata(new Utf8("canonical"));
+    assertEquals(parse.getText(), "");
+    assertEquals(parse.getTitle(), "");
+    assertEquals(parse.getParseStatus().getMajorCode(), ParseStatusCodes.FAILED);
+    assertEquals(parse.getParseStatus().getMinorCode(), 999);
+    assertNotNull(wp.getFromMetadata(new Utf8("canonical")));
+    assertEquals(Bytes.toString(urlFromMetadata), canonicalUrl);
+  }
+
+  /**
+   * Prepare Webpage to pass HtmlParser
+   * 
+   * @param baseUrl
+   * @param content
+   * @return
+   */
+  public WebPage prepareWebpage(String baseUrl, String content) {
+    WebPage wp = new WebPage();
+    wp.setContent(ByteBuffer.wrap(Bytes.toBytes(content)));
+    wp.setBaseUrl(new Utf8(baseUrl));
+    wp.setText(new Utf8(""));
+    wp.setTitle(new Utf8(""));
+    wp.setStatus(ParseStatusCodes.FAILED);
+    return wp;
+  }
+
+  /**
+   * Test Baseurl with canonical url equality.
+   * 
+   * @param node
+   * @param url
+   * @throws MalformedURLException
+   */
+  private void compareCanonicalLink(DocumentFragment node, URL url)
+      throws MalformedURLException {
+    assertEquals(utils.getDifferentCanonical(node, url), new URL(canonicalUrl));
+  }
+
 
   private static final void appendOutlinks(StringBuffer sb, Outlink[] o) {
     for (int i= 0; i < o.length; i++) {
diff --git src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
index 06dadff..b3fa0d7 100644
--- src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
+++ src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
@@ -26,6 +26,8 @@ import java.util.HashMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.nutch.parse.Outlink;
 import org.apache.nutch.util.NodeWalker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.w3c.dom.NamedNodeMap;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
@@ -38,6 +40,8 @@ import org.w3c.dom.NodeList;
  *
  */
 public class DOMContentUtils {
+  public static final Logger LOG = LoggerFactory
+      .getLogger("org.apache.nutch.parse.tika");
 
   private static class LinkParams {
 	private String elName;
@@ -412,6 +416,46 @@ public class DOMContentUtils {
       }
     }
   }
+  // Better to pass urls throught regexp url normalizer before comparing?
+  private String normalizeUrl(String url) {
+    return url.replaceFirst("/$", "");
+  }
+
+  /**
+   * Check if page supplied in DOM <code>node</code> has canonical link element,
+   * if so compare it to page <code>url</code> if those are different then page
+   * is not canonical.
+   */
+
+  public URL getDifferentCanonical(Node node, URL url) {
+    NodeWalker walker = new NodeWalker(node);
+    while (walker.hasNext()) {
+
+      Node currentNode = walker.nextNode();
+      String nodeName = currentNode.getNodeName();
+      if ("link".equalsIgnoreCase(nodeName)) {
+        NamedNodeMap attrs = currentNode.getAttributes();
+        Node rel = attrs.getNamedItem("rel");
+        if (rel != null && "canonical".equalsIgnoreCase(rel.getNodeValue())) {
+          String href = attrs.getNamedItem("href").getNodeValue();
+          try {
+            URL canonical = new URL(href);
+            if (!checkUrlEquality(canonical, url)) {
+              return canonical;
+            }
+          } catch (MalformedURLException e) {
+            LOG.warn("There is an error.When check canonical and base url equaltiy.");
+          }
+        }
+      }
+    }
+    return null;
+  }
 
+  public boolean checkUrlEquality(URL canonical, URL url) {
+    return normalizeUrl(canonical.getFile()).equalsIgnoreCase(
+        normalizeUrl(url.getFile()))
+        && canonical.getHost().equalsIgnoreCase(url.getHost());
+  }
 }
 
diff --git src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
index 52e0841..303b376 100644
--- src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
+++ src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
@@ -70,7 +70,7 @@ public class TikaParser implements org.apache.nutch.parse.Parser {
     FIELDS.add(WebPage.Field.BASE_URL);
     FIELDS.add(WebPage.Field.CONTENT_TYPE);
   }
-
+  private static final String CANONICAL_VALUE = "canonical";
   private Configuration conf;
   private TikaConfig tikaConfig = null;
   private DOMContentUtils utils;
@@ -162,6 +162,12 @@ public class TikaParser implements org.apache.nutch.parse.Parser {
         LOG.trace("found " + outlinks.length + " outlinks in " + base);
       }
     }
+    // get and control canonical(rel="canonical") value.
+    URL canonical = utils.getDifferentCanonical(root, base);
+    if (canonical != null) {
+      LOG.warn("Non canonical page. Ignore: " + base);
+      return signCanonical(canonical, page);
+    }
 
     // populate Nutch metadata with Tika metadata
     String[] TikaMDNames = tikamd.names();
@@ -198,6 +204,13 @@ public class TikaParser implements org.apache.nutch.parse.Parser {
     return parse;
   }
 
+  // write canonical url to metadata and return Empty Parse.
+  public Parse signCanonical(URL base, WebPage wp) {
+    wp.putToMetadata(new Utf8(CANONICAL_VALUE),
+        ByteBuffer.wrap(base.toString().getBytes()));
+    return ParseStatusUtils.getEmptyParse(999, "Non canonical page. Ignore",
+        getConf());
+  }
   public void setConf(Configuration conf) {
     this.conf = conf;
     this.tikaConfig = null;
diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
deleted file mode 100644
index 5d80125..0000000
--- src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nutch.parse.tika;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-import org.apache.nutch.parse.Outlink;
-import org.apache.nutch.parse.tika.DOMBuilder;
-import org.apache.nutch.parse.tika.DOMContentUtils;
-import org.apache.nutch.parse.tika.TikaParser;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.html.dom.HTMLDocumentImpl;
-import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.xml.serializer.dom3.LSSerializerImpl;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
-
-import java.io.ByteArrayInputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.StringTokenizer;
-
-/**
- * Unit tests for DOMContentUtils.
- */
-public class DOMContentUtilsTest {
-
-	private static final String[] testPages = {
-			// 0.
-			new String(
-					"<html><head><title> title </title><script> script </script>"
-							+ "</head><body> body <a href=\"http://www.nutch.org\">"
-							+ " anchor </a><!--comment-->" + "</body></html>"),
-			// 1.
-			new String(
-					"<html><head><title> title </title><script> script </script>"
-							+ "</head><body> body <a href=\"/\">"
-							+ " home </a><!--comment-->"
-							+ "<style> style </style>"
-							+ " <a href=\"bot.html\">" + " bots </a>"
-							+ "</body></html>"),
-			// 2.
-			new String("<html><head><title> </title>" + "</head><body> "
-					+ "<a href=\"/\"> separate this "
-					+ "<a href=\"ok\"> from this" + "</a></a>"
-					+ "</body></html>"),
-			// 3.
-			// this one relies on certain neko fixup behavior, possibly
-			// distributing the anchors into the LI's-but not the other
-			// anchors (outside of them, instead)! So you get a tree that
-			// looks like:
-			// ... <li> <a href=/> home </a> </li>
-			// <li> <a href=/> <a href="1"> 1 </a> </a> </li>
-			// <li> <a href=/> <a href="1"> <a href="2"> 2 </a> </a> </a> </li>
-			new String("<html><head><title> my title </title>"
-					+ "</head><body> body " + "<ul>"
-					+ "<li> <a href=\"/\"> home" + "<li> <a href=\"1\"> 1"
-					+ "<li> <a href=\"2\"> 2" + "</ul>" + "</body></html>"),
-			// 4.
-			// test frameset link extraction. The invalid frame in the middle
-			// will be
-			// fixed to a third standalone frame.
-			new String("<html><head><title> my title </title>"
-					+ "</head><frameset rows=\"20,*\"> "
-					+ "<frame src=\"top.html\">" + "</frame>"
-					+ "<frameset cols=\"20,*\">" + "<frame src=\"left.html\">"
-					+ "</frame>" + "<frame src=\"invalid.html\"/>" + "</frame>"
-					+ "<frame src=\"right.html\">" + "</frame>" + "</frameset>"
-					+ "</frameset>" + "</body></html>"),
-			// 5.
-			// test <area> and <iframe> link extraction + url normalization
-			new String(
-					"<html><head><title> my title </title>"
-							+ "</head><body>"
-							+ "<img src=\"logo.gif\" usemap=\"#green\" border=\"0\">"
-							+ "<map name=\"green\">"
-							+ "<area shape=\"polygon\" coords=\"19,44,45,11,87\" href=\"../index.html\">"
-							+ "<area shape=\"rect\" coords=\"128,132,241,179\" href=\"#bottom\">"
-							+ "<area shape=\"circle\" coords=\"68,211,35\" href=\"../bot.html\">"
-							+ "</map>"
-							+ "<a name=\"bottom\"/><h1> the bottom </h1> "
-							+ "<iframe src=\"../docs/index.html\"/>"
-							+ "</body></html>"),
-			// 6.
-			// test whitespace processing for plain text extraction
-			new String(
-					"<html><head>\n <title> my\t\n  title\r\n </title>\n"
-							+ " </head>\n"
-							+ " <body>\n"
-							+ "    <h1> Whitespace\ttest  </h1> \n"
-							+ "\t<a href=\"../index.html\">\n  \twhitespace  test\r\n\t</a>  \t\n"
-							+ "    <p> This is<span> a whitespace<span></span> test</span>. Newlines\n"
-							+ "should appear as space too.</p><p>Tabs\tare spaces too.\n</p>"
-							+ "    This\t<b>is a</b> break -&gt;<br>and the line after<i> break</i>.<br>\n"
-							+ "<table>"
-							+ "    <tr><td>one</td><td>two</td><td>three</td></tr>\n"
-							+ "    <tr><td>space here </td><td> space there</td><td>no space</td></tr>"
-							+ "\t<tr><td>one\r\ntwo</td><td>two\tthree</td><td>three\r\tfour</td></tr>\n"
-							+ "</table>put some text here<Br>and there."
-							+ "<h2>End\tthis\rmadness\n!</h2>\r\n"
-							+ "         .        .        .         ."
-							+ "</body>  </html>"),
-			// 7.
-			// test that <a rel=nofollow> links are not returned
-			new String(
-					"<html><head></head><body>"
-							+ "<a href=\"http://www.nutch.org\" rel=\"nofollow\"> ignore </a>"
-							+ "<a rel=\"nofollow\" href=\"http://www.nutch.org\"> ignore </a>"
-							+ "</body></html>"),
-			// 8.
-			// test that POST form actions are skipped
-			new String(
-					"<html><head></head><body>"
-							+ "<form method='POST' action='/search.jsp'><input type=text>"
-							+ "<input type=submit><p>test1</p></form>"
-							+ "<form method='GET' action='/dummy.jsp'><input type=text>"
-							+ "<input type=submit><p>test2</p></form></body></html>"),
-			// 9.
-			// test that all form actions are skipped
-			new String(
-					"<html><head></head><body>"
-							+ "<form method='POST' action='/search.jsp'><input type=text>"
-							+ "<input type=submit><p>test1</p></form>"
-							+ "<form method='GET' action='/dummy.jsp'><input type=text>"
-							+ "<input type=submit><p>test2</p></form></body></html>"),
-			// 10.
-			new String("<html><head><title> title </title>" + "</head><body>"
-					+ "<a href=\";x\">anchor1</a>"
-					+ "<a href=\"g;x\">anchor2</a>"
-					+ "<a href=\"g;x?y#s\">anchor3</a>" + "</body></html>"),
-			// 11.
-			new String("<html><head><title> title </title>" + "</head><body>"
-					+ "<a href=\"g\">anchor1</a>"
-					+ "<a href=\"g?y#s\">anchor2</a>"
-					+ "<a href=\"?y=1\">anchor3</a>"
-					+ "<a href=\"?y=1#s\">anchor4</a>"
-					+ "<a href=\"?y=1;somethingelse\">anchor5</a>"
-					+ "</body></html>"), };
-
-	private static int SKIP = 9;
-
-	private static String[] testBaseHrefs = { "http://www.nutch.org",
-			"http://www.nutch.org/docs/foo.html", "http://www.nutch.org/docs/",
-			"http://www.nutch.org/docs/", "http://www.nutch.org/frames/",
-			"http://www.nutch.org/maps/", "http://www.nutch.org/whitespace/",
-			"http://www.nutch.org//", "http://www.nutch.org/",
-			"http://www.nutch.org/", "http://www.nutch.org/",
-			"http://www.nutch.org/;something" };
-
-	private static final DocumentFragment testDOMs[] = new DocumentFragment[testPages.length];
-
-	private static URL[] testBaseHrefURLs = new URL[testPages.length];
-
-	private static final String[] answerText = {
-			"body anchor",
-			"body home bots",
-			"separate this from this",
-			"body home 1 2",
-			"",
-			"the bottom",
-			"Whitespace test whitespace test "
-					+ "This is a whitespace test . Newlines should appear as space too. "
-					+ "Tabs are spaces too. This is a break -> and the line after break . "
-					+ "one two three space here space there no space "
-					+ "one two two three three four put some text here and there. "
-					+ "End this madness ! . . . .", "ignore ignore",
-			"test1 test2", "test1 test2", "anchor1 anchor2 anchor3",
-			"anchor1 anchor2 anchor3 anchor4 anchor5" };
-
-	private static final String[] answerTitle = { "title", "title", "",
-			"my title", "my title", "my title", "my title", "", "", "",
-			"title", "title" };
-
-	// note: should be in page-order
-	private static Outlink[][] answerOutlinks;
-
-	private static Configuration conf;
-	private static DOMContentUtils utils = null;
-	
-	public static final Logger Logger = LoggerFactory.getLogger(DOMContentUtilsTest.class);
-
-	public DOMContentUtilsTest(String name) {
-	}
-
-	private static void setup() throws Exception {
-		conf = NutchConfiguration.create();
-		conf.setBoolean("parser.html.form.use_action", true);
-		utils = new DOMContentUtils(conf);
-		TikaParser tikaParser = new TikaParser();
-		tikaParser.setConf(conf);
-		Parser parser = tikaParser.getTikaConfig().getParser("text/html");
-		for (int i = 0; i < testPages.length; i++) {
-			Metadata tikamd = new Metadata();
-
-			HTMLDocumentImpl doc = new HTMLDocumentImpl();
-			doc.setErrorChecking(false);
-			DocumentFragment root = doc.createDocumentFragment();
-			DOMBuilder domhandler = new DOMBuilder(doc, root);
-			ParseContext context = new ParseContext();
-			// to add once available in Tika
-			//context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
-			try {
-				parser.parse(new ByteArrayInputStream(testPages[i].getBytes()),
-						domhandler, tikamd, context);
-				testBaseHrefURLs[i] = new URL(testBaseHrefs[i]);
-			} catch (Exception e) {
-				e.printStackTrace();
-				fail("caught exception: " + e);
-			}
-			testDOMs[i] = root;
-			LSSerializerImpl lsi = new LSSerializerImpl();
-			System.out.println("input " + i + ": '" + testPages[i] + "'");
-			System.out.println("output " + i + ": '" + lsi.writeToString(root)
-					+ "'");
-
-		}
-		answerOutlinks = new Outlink[][] {
-				// 0
-				{ new Outlink("http://www.nutch.org", "anchor"), },
-				// 1
-				{
-				  new Outlink("http://www.nutch.org/", "home"),
-				  new Outlink("http://www.nutch.org/docs/bot.html",
-								"bots"), },
-				// 2
-				{
-					new Outlink("http://www.nutch.org/", "separate this"),
-					new Outlink("http://www.nutch.org/docs/ok", "from this"), },
-				
-				// 3	
-				{   new Outlink("http://www.nutch.org/", "home"),
-					new Outlink("http://www.nutch.org/docs/1", "1"),
-					new Outlink("http://www.nutch.org/docs/2", "2"), },
-				// 4	
-				{
-					new Outlink("http://www.nutch.org/frames/top.html", ""),
-					new Outlink("http://www.nutch.org/frames/left.html", ""),
-					new Outlink("http://www.nutch.org/frames/invalid.html",""),
-					new Outlink("http://www.nutch.org/frames/right.html",""), 
-				},
-				// 5
-				{ 
-					new Outlink("http://www.nutch.org/maps/logo.gif", ""),
-					new Outlink("http://www.nutch.org/index.html", ""),
-					new Outlink("http://www.nutch.org/maps/#bottom", ""),
-					new Outlink("http://www.nutch.org/bot.html", ""),
-					new Outlink("http://www.nutch.org/docs/index.html", "") 
-				},
-				// 6
-				{ new Outlink("http://www.nutch.org/index.html",
-						"whitespace test"), 
-				},
-				// 7
-				{},
-				// 8
-				{ new Outlink("http://www.nutch.org/dummy.jsp", "test2"), },
-				// 9
-				{},
-				// 10 
-				{ 
-				 new Outlink("http://www.nutch.org/;x", "anchor1"),
-				 new Outlink("http://www.nutch.org/g;x", "anchor2"),
-				 new Outlink("http://www.nutch.org/g;x?y#s", "anchor3") 
-				},
-				// 11
-				{
-				 new Outlink("http://www.nutch.org/g;something","anchor1"),
-				 new Outlink("http://www.nutch.org/g;something?y#s", "anchor2"),
-				 new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
-				 new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
-				 new Outlink("http://www.nutch.org/?y=1;somethingelse", "anchor5") }
-				};
-
-	}
-
-	private static boolean equalsIgnoreWhitespace(String s1, String s2) {
-		StringTokenizer st1 = new StringTokenizer(s1);
-		StringTokenizer st2 = new StringTokenizer(s2);
-
-		while (st1.hasMoreTokens()) {
-			if (!st2.hasMoreTokens()) {
-			 Logger.info("st1+ '" + st1.nextToken() + "'");
-				return false;
-			}
-			String st1Token = st1.nextToken();
-			String st2Token = st2.nextToken();
-			if (!st1Token.equals(st2Token)) {
-			 Logger.info("st1:'" + st1Token + "' != st2:'" + st2Token + "'");
-				return false;
-			}
-		}
-		if (st2.hasMoreTokens()) {
-			System.err.println("st2+ '" + st2.nextToken() + "'");
-			return false;
-		}
-		return true;
-	}
-
-	@Test
-	public void testGetText() throws Exception {
-		if (testDOMs[0] == null)
-			setup();
-		for (int i = 0; i < testPages.length; i++) {
-			StringBuffer sb = new StringBuffer();
-			utils.getText(sb, testDOMs[i]);
-			String text = sb.toString();
-			assertTrue(
-					"example " + i + " : expecting text: " + answerText[i]
-							+ System.getProperty("line.separator")
-							+ System.getProperty("line.separator")
-							+ "got text: " + text,
-					equalsIgnoreWhitespace(answerText[i], text));
-		}
-	}
-
-	// won't work with Tika - the title is stored in the metadata but
-	// not put in the XHTML representation
-	@Test
-	public void testGetTitle() throws Exception {
-		if (testDOMs[0] == null)
-			setup();
-		for (int i = 0; i < testPages.length; i++) {
-			StringBuffer sb = new StringBuffer();
-			utils.getTitle(sb, testDOMs[i]);
-			String title = sb.toString();
-			assertTrue(
-					"example " + i + " : expecting title: " + answerTitle[i]
-							+ System.getProperty("line.separator")
-							+ System.getProperty("line.separator")
-							+ "got title: " + title,
-					equalsIgnoreWhitespace(answerTitle[i], title));
-		}
-	}
-
-	@Test
-	public void testGetOutlinks() throws Exception {
-		if (testDOMs[0] == null)
-			setup();
-		for (int i = 0; i < testPages.length; i++) {
-			ArrayList<Outlink> outlinks = new ArrayList<Outlink>();
-			if (i == SKIP) {
-				conf.setBoolean("parser.html.form.use_action", false);
-				utils.setConf(conf);
-			} else {
-				conf.setBoolean("parser.html.form.use_action", true);
-				utils.setConf(conf);
-			}
-			utils.getOutlinks(testBaseHrefURLs[i], outlinks, testDOMs[i]);
-			Outlink[] outlinkArr = new Outlink[outlinks.size()];
-			outlinkArr = outlinks.toArray(outlinkArr);
-			compareOutlinks(i, answerOutlinks[i], outlinkArr);
-		}
-	}
-
-	private static final void appendOutlinks(StringBuffer sb, Outlink[] o) {
-		for (int i = 0; i < o.length; i++) {
-			sb.append(o[i].toString());
-			sb.append(System.getProperty("line.separator"));
-		}
-	}
-
-	private static final String outlinksString(Outlink[] o) {
-		StringBuffer sb = new StringBuffer();
-		appendOutlinks(sb, o);
-		return sb.toString();
-	}
-
-	private static final void compareOutlinks(int test, Outlink[] o1,
-			Outlink[] o2) {
-		if (o1.length != o2.length) {
-			assertTrue(
-					"test " + test
-							+ ", got wrong number of outlinks (expecting "
-							+ o1.length + ", got " + o2.length + ")"
-							+ System.getProperty("line.separator") + "answer: "
-							+ System.getProperty("line.separator")
-							+ outlinksString(o1)
-							+ System.getProperty("line.separator") + "got: "
-							+ System.getProperty("line.separator")
-							+ outlinksString(o2)
-							+ System.getProperty("line.separator"), false);
-		}
-
-		for (int i = 0; i < o1.length; i++) {
-			if (!o1[i].equals(o2[i])) {
-				assertTrue(
-						"test " + test + ", got wrong outlinks at position "
-								+ i + System.getProperty("line.separator")
-								+ "answer: "
-								+ System.getProperty("line.separator")
-								+ o1[i].toString()
-								+ System.getProperty("line.separator")
-								+ "got: "
-								+ System.getProperty("line.separator")
-								+ o2[i].toString(), false);
-
-			}
-		}
-	}
-}
diff --git src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestTikaDOMContentUtils.java src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestTikaDOMContentUtils.java
new file mode 100644
index 0000000..13efdaf
--- /dev/null
+++ src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestTikaDOMContentUtils.java
@@ -0,0 +1,490 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.parse.tika;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+import org.apache.nutch.parse.Outlink;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseStatusCodes;
+import org.apache.nutch.parse.tika.DOMBuilder;
+import org.apache.nutch.parse.tika.DOMContentUtils;
+import org.apache.nutch.parse.tika.TikaParser;
+import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.html.dom.HTMLDocumentImpl;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.Bytes;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.xml.serializer.dom3.LSSerializerImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.DocumentFragment;
+
+import java.io.ByteArrayInputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+
+/**
+ * Unit tests for DOMContentUtils.
+ */
+public class TestTikaDOMContentUtils {
+  public static final Logger LOG = LoggerFactory
+      .getLogger("org.apache.nutch.parse.tika");
+
+  private static final String[] testPages = {
+      // 0.
+      new String("<html><head><title> title </title><script> script </script>"
+          + "</head><body> body <a href=\"http://www.nutch.org\">"
+          + " anchor </a><!--comment-->" + "</body></html>"),
+      // 1.
+      new String("<html><head><title> title </title><script> script </script>"
+          + "</head><body> body <a href=\"/\">" + " home </a><!--comment-->"
+          + "<style> style </style>" + " <a href=\"bot.html\">" + " bots </a>"
+          + "</body></html>"),
+      // 2.
+      new String("<html><head><title> </title>" + "</head><body> "
+          + "<a href=\"/\"> separate this " + "<a href=\"ok\"> from this"
+          + "</a></a>" + "</body></html>"),
+      // 3.
+      // this one relies on certain neko fixup behavior, possibly
+      // distributing the anchors into the LI's-but not the other
+      // anchors (outside of them, instead)! So you get a tree that
+      // looks like:
+      // ... <li> <a href=/> home </a> </li>
+      // <li> <a href=/> <a href="1"> 1 </a> </a> </li>
+      // <li> <a href=/> <a href="1"> <a href="2"> 2 </a> </a> </a> </li>
+      new String("<html><head><title> my title </title>"
+          + "</head><body> body " + "<ul>" + "<li> <a href=\"/\"> home"
+          + "<li> <a href=\"1\"> 1" + "<li> <a href=\"2\"> 2" + "</ul>"
+          + "</body></html>"),
+      // 4.
+      // test frameset link extraction. The invalid frame in the middle
+      // will be
+      // fixed to a third standalone frame.
+      new String("<html><head><title> my title </title>"
+          + "</head><frameset rows=\"20,*\"> " + "<frame src=\"top.html\">"
+          + "</frame>" + "<frameset cols=\"20,*\">"
+          + "<frame src=\"left.html\">" + "</frame>"
+          + "<frame src=\"invalid.html\"/>" + "</frame>"
+          + "<frame src=\"right.html\">" + "</frame>" + "</frameset>"
+          + "</frameset>" + "</body></html>"),
+      // 5.
+      // test <area> and <iframe> link extraction + url normalization
+      new String(
+          "<html><head><title> my title </title>"
+              + "</head><body>"
+              + "<img src=\"logo.gif\" usemap=\"#green\" border=\"0\">"
+              + "<map name=\"green\">"
+              + "<area shape=\"polygon\" coords=\"19,44,45,11,87\" href=\"../index.html\">"
+              + "<area shape=\"rect\" coords=\"128,132,241,179\" href=\"#bottom\">"
+              + "<area shape=\"circle\" coords=\"68,211,35\" href=\"../bot.html\">"
+              + "</map>" + "<a name=\"bottom\"/><h1> the bottom </h1> "
+              + "<iframe src=\"../docs/index.html\"/>" + "</body></html>"),
+      // 6.
+      // test whitespace processing for plain text extraction
+      new String(
+          "<html><head>\n <title> my\t\n  title\r\n </title>\n"
+              + " </head>\n"
+              + " <body>\n"
+              + "    <h1> Whitespace\ttest  </h1> \n"
+              + "\t<a href=\"../index.html\">\n  \twhitespace  test\r\n\t</a>  \t\n"
+              + "    <p> This is<span> a whitespace<span></span> test</span>. Newlines\n"
+              + "should appear as space too.</p><p>Tabs\tare spaces too.\n</p>"
+              + "    This\t<b>is a</b> break -&gt;<br>and the line after<i> break</i>.<br>\n"
+              + "<table>"
+              + "    <tr><td>one</td><td>two</td><td>three</td></tr>\n"
+              + "    <tr><td>space here </td><td> space there</td><td>no space</td></tr>"
+              + "\t<tr><td>one\r\ntwo</td><td>two\tthree</td><td>three\r\tfour</td></tr>\n"
+              + "</table>put some text here<Br>and there."
+              + "<h2>End\tthis\rmadness\n!</h2>\r\n"
+              + "         .        .        .         ." + "</body>  </html>"),
+      // 7.
+      // test that <a rel=nofollow> links are not returned
+      new String("<html><head></head><body>"
+          + "<a href=\"http://www.nutch.org\" rel=\"nofollow\"> ignore </a>"
+          + "<a rel=\"nofollow\" href=\"http://www.nutch.org\"> ignore </a>"
+          + "</body></html>"),
+      // 8.
+      // test that POST form actions are skipped
+      new String("<html><head></head><body>"
+          + "<form method='POST' action='/search.jsp'><input type=text>"
+          + "<input type=submit><p>test1</p></form>"
+          + "<form method='GET' action='/dummy.jsp'><input type=text>"
+          + "<input type=submit><p>test2</p></form></body></html>"),
+      // 9.
+      // test that all form actions are skipped
+      new String("<html><head></head><body>"
+          + "<form method='POST' action='/search.jsp'><input type=text>"
+          + "<input type=submit><p>test1</p></form>"
+          + "<form method='GET' action='/dummy.jsp'><input type=text>"
+          + "<input type=submit><p>test2</p></form></body></html>"),
+
+      // 10.Also test canonical
+      new String(
+          "<html><head><title> title </title>"
+              + "<link rel=\"canonical\" href=\"http://www.canonical.com/\">canonical</a>"
+              + "</head><body>" + "<a href=\";x\">anchor1</a>"
+              + "<a href=\"g;x\">anchor2</a>"
+              + "<a href=\"g;x?y#s\">anchor3</a>" + "</body></html>"),
+      // 11.
+      new String("<html><head><title> title </title>" + "</head><body>"
+          + "<a href=\"g\">anchor1</a>" + "<a href=\"g?y#s\">anchor2</a>"
+          + "<a href=\"?y=1\">anchor3</a>" + "<a href=\"?y=1#s\">anchor4</a>"
+          + "<a href=\"?y=1;somethingelse\">anchor5</a>" + "</body></html>"), };
+
+  private static int SKIP = 9;
+
+  private static String[] testBaseHrefs = { "http://www.nutch.org",
+      "http://www.nutch.org/docs/foo.html", "http://www.nutch.org/docs/",
+      "http://www.nutch.org/docs/", "http://www.nutch.org/frames/",
+      "http://www.nutch.org/maps/", "http://www.nutch.org/whitespace/",
+      "http://www.nutch.org//", "http://www.nutch.org/",
+      "http://www.nutch.org/",
+      // Also baseurl to test canonical
+      "http://www.nutch.org/", "http://www.nutch.org/;something" };
+
+  private static final DocumentFragment testDOMs[] = new DocumentFragment[testPages.length];
+
+  private static URL[] testBaseHrefURLs = new URL[testPages.length];
+
+  private static final String[] answerText = {
+      "title body anchor",
+      "title body home bots",
+      "separate this from this",
+      "my title body home 1 2",
+      "my title",
+      "my title the bottom",
+      "my title Whitespace test whitespace test This is a whitespace test. "
+          + "Newlines should appear as space too. Tabs are spaces too. "
+          + "This is a break -> and the line after break. "
+          + "one two three space here space there no space one two two three three four put some text here and there."
+          + " End this madness ! . . . .", "ignore ignore", "test1 test2",
+      "test1 test2", "title canonical anchor1 anchor2 anchor3 ",
+      "title anchor1 anchor2 anchor3 anchor4 anchor5" };
+
+  private static final String[] answerTitle = { "title", "title", "",
+      "my title", "my title", "my title", "my title", "", "", "", "title",
+      "title" };
+
+  // note: should be in page-order
+  private static Outlink[][] answerOutlinks;
+
+  private static Configuration conf;
+  private static DOMContentUtils utils = null;
+
+  public static final Logger Logger = LoggerFactory
+      .getLogger(TestTikaDOMContentUtils.class);
+
+  private Parse parsetp;
+
+  private String contentType = "text/html";
+
+  private String canonicalUrl = "http://www.canonical.com/";
+
+  public TestTikaDOMContentUtils() {
+  }
+
+  private static void setup() throws Exception {
+    conf = NutchConfiguration.create();
+    conf.setBoolean("parser.html.form.use_action", true);
+    utils = new DOMContentUtils(conf);
+    TikaParser tikaParser = new TikaParser();
+    tikaParser.setConf(conf);
+    Parser parser = tikaParser.getTikaConfig().getParser("text/html");
+    for (int i = 0; i < testPages.length; i++) {
+      Metadata tikamd = new Metadata();
+
+      HTMLDocumentImpl doc = new HTMLDocumentImpl();
+      doc.setErrorChecking(false);
+      DocumentFragment root = doc.createDocumentFragment();
+      DOMBuilder domhandler = new DOMBuilder(doc, root);
+      ParseContext context = new ParseContext();
+      // to add once available in Tika
+      // context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE);
+      try {
+        parser.parse(new ByteArrayInputStream(testPages[i].getBytes()),
+            domhandler, tikamd, context);
+        testBaseHrefURLs[i] = new URL(testBaseHrefs[i]);
+      } catch (Exception e) {
+        e.printStackTrace();
+        fail("caught exception: " + e);
+      }
+      testDOMs[i] = root;
+      LSSerializerImpl lsi = new LSSerializerImpl();
+      System.out.println("input " + i + ": '" + testPages[i] + "'");
+      System.out.println("output " + i + ": '" + lsi.writeToString(root) + "'");
+
+    }
+    answerOutlinks = new Outlink[][] {
+        // 0
+        { new Outlink("http://www.nutch.org", "anchor"), },
+        // 1
+        { new Outlink("http://www.nutch.org/", "home"),
+            new Outlink("http://www.nutch.org/docs/bot.html", "bots"), },
+        // 2
+        { new Outlink("http://www.nutch.org/", "separate this"),
+            new Outlink("http://www.nutch.org/docs/ok", "from this"), },
+
+        // 3
+        { new Outlink("http://www.nutch.org/", "home"),
+            new Outlink("http://www.nutch.org/docs/1", "1"),
+            new Outlink("http://www.nutch.org/docs/2", "2"), },
+        // 4
+        { new Outlink("http://www.nutch.org/frames/top.html", ""),
+            new Outlink("http://www.nutch.org/frames/left.html", ""),
+            new Outlink("http://www.nutch.org/frames/invalid.html", ""),
+            new Outlink("http://www.nutch.org/frames/right.html", ""), },
+        // 5
+        { new Outlink("http://www.nutch.org/maps/logo.gif", ""),
+            new Outlink("http://www.nutch.org/index.html", ""),
+            new Outlink("http://www.nutch.org/maps/#bottom", ""),
+            new Outlink("http://www.nutch.org/bot.html", ""),
+            new Outlink("http://www.nutch.org/docs/index.html", "") },
+        // 6
+        { new Outlink("http://www.nutch.org/index.html", "whitespace test"), },
+        // 7
+        {},
+        // 8
+        { // Tika would not find "form" tag.?
+          // new Outlink("http://www.nutch.org/dummy.jsp", "test2"),
+        },
+
+        // 9
+        {},
+        // 10
+        { new Outlink("http://www.canonical.com/", ""),
+            new Outlink("http://www.nutch.org/;x", "anchor1"),
+            new Outlink("http://www.nutch.org/g;x", "anchor2"),
+            new Outlink("http://www.nutch.org/g;x?y#s", "anchor3") },
+        // 11
+        {
+
+        new Outlink("http://www.nutch.org/g;something", "anchor1"),
+            new Outlink("http://www.nutch.org/g;something?y#s", "anchor2"),
+            new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
+            new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
+            new Outlink("http://www.nutch.org/?y=1;somethingelse", "anchor5") } };
+
+  }
+
+  private static boolean equalsIgnoreWhitespace(String s1, String s2) {
+    StringTokenizer st1 = new StringTokenizer(s1);
+    StringTokenizer st2 = new StringTokenizer(s2);
+
+    while (st1.hasMoreTokens()) {
+      if (!st2.hasMoreTokens()) {
+        Logger.info("st1+ '" + st1.nextToken() + "'");
+        return false;
+      }
+      String st1Token = st1.nextToken();
+      String st2Token = st2.nextToken();
+      if (!st1Token.equals(st2Token)) {
+        Logger.info("st1:'" + st1Token + "' != st2:'" + st2Token + "'");
+        return false;
+      }
+    }
+    if (st2.hasMoreTokens()) {
+      System.err.println("st2+ '" + st2.nextToken() + "'");
+      return false;
+    }
+    return true;
+  }
+
+  @Test
+  public void testGetText() throws Exception {
+    if (testDOMs[0] == null)
+      setup();
+    for (int i = 0; i < testPages.length; i++) {
+      StringBuffer sb = new StringBuffer();
+      utils.getText(sb, testDOMs[i]);
+      String text = sb.toString();
+      assertTrue(
+          "example " + i + " : expecting text: " + answerText[i]
+              + System.getProperty("line.separator")
+              + System.getProperty("line.separator") + "got text: " + text,
+          equalsIgnoreWhitespace(answerText[i], text));
+    }
+  }
+
+  // won't work with Tika - the title is stored in the metadata but
+  // not put in the XHTML representation
+  @Test
+  public void testGetTitle() throws Exception {
+    if (testDOMs[0] == null)
+      setup();
+    for (int i = 0; i < testPages.length; i++) {
+      StringBuffer sb = new StringBuffer();
+      utils.getTitle(sb, testDOMs[i]);
+      String title = sb.toString();
+      assertTrue(
+          "example " + i + " : expecting title: " + answerTitle[i]
+              + System.getProperty("line.separator")
+              + System.getProperty("line.separator") + "got title: " + title,
+          equalsIgnoreWhitespace(answerTitle[i], title));
+    }
+  }
+
+  @Test
+  public void testGetOutlinks() throws Exception {
+    if (testDOMs[0] == null)
+      setup();
+    for (int i = 0; i < testPages.length; i++) {
+      ArrayList<Outlink> outlinks = new ArrayList<Outlink>();
+      if (i == SKIP) {
+        conf.setBoolean("parser.html.form.use_action", false);
+        utils.setConf(conf);
+      } else {
+        conf.setBoolean("parser.html.form.use_action", true);
+        utils.setConf(conf);
+      }
+      utils.getOutlinks(testBaseHrefURLs[i], outlinks, testDOMs[i]);
+
+      Outlink[] outlinkArr = new Outlink[outlinks.size()];
+      outlinkArr = outlinks.toArray(outlinkArr);
+      compareOutlinks(i, answerOutlinks[i], outlinkArr);
+    }
+  }
+
+  /**
+   * 
+   * TestCase to test Canonical situations.
+   */
+  @Test
+  public void testCanonical() {
+    if (testDOMs[0] == null)
+      try {
+        setup();
+      } catch (Exception e1) {
+        e1.printStackTrace();
+      }
+    for (int i = 0; i < testPages.length; i++) {
+      if (i == 10) {
+        try {
+          compareCanonicalLink(testDOMs[i], new URL(testBaseHrefs[i]));
+          compareCanonicalParse(testBaseHrefs[i], testPages[i]);
+        } catch (MalformedURLException e) {
+          LOG.warn("There is an error.When test canonical link or Parsed canonical page.");
+        }
+      } else
+        continue;
+    }
+  }
+
+  /**
+   * 
+   * If url of page which has canonical tag not equals href attribute of
+   * value.And canonical enables via confugartion. Should return empty Parse.
+   * Method compares conclusions of Parse.
+   * 
+   * @param baseUrl
+   * @param content
+   */
+  public void compareCanonicalParse(String baseUrl, String content) {
+    WebPage wp = prepareWebpage(baseUrl, content);
+    TikaParser tp = new TikaParser();
+    tp.setConf(conf);
+    parsetp = tp.getParse(baseUrl, wp);
+    ByteBuffer urlFromMetadata = wp.getFromMetadata(new Utf8("canonical"));
+    assertEquals(parsetp.getText(), "");
+    assertEquals(parsetp.getTitle(), "");
+    assertEquals(parsetp.getParseStatus().getMajorCode(),
+        ParseStatusCodes.FAILED);
+    assertEquals(parsetp.getParseStatus().getMinorCode(), 999);
+    assertNotNull(wp.getFromMetadata(new Utf8("canonical")));
+    assertEquals(Bytes.toString(urlFromMetadata), canonicalUrl);
+  }
+
+  /**
+   * Prepare Webpage to pass TikaParser
+   * 
+   * @param baseUrl
+   * @param content
+   * @return
+   */
+  public WebPage prepareWebpage(String baseUrl, String content) {
+    WebPage wp = new WebPage();
+    wp.setContent(ByteBuffer.wrap(Bytes.toBytes(content)));
+    wp.setContentType(new Utf8(contentType));
+    wp.setBaseUrl(new Utf8(baseUrl));
+    wp.setText(new Utf8(""));
+    wp.setTitle(new Utf8(""));
+    wp.setStatus(ParseStatusCodes.FAILED);
+
+    return wp;
+  }
+
+  /**
+   * Test Baseurl with canonical url equality.
+   * 
+   * @param node
+   * @param url
+   * @throws MalformedURLException
+   */
+  private void compareCanonicalLink(DocumentFragment node, URL url)
+      throws MalformedURLException {
+    assertEquals(utils.getDifferentCanonical(node, url), new URL(canonicalUrl));
+  }
+
+  private static final void appendOutlinks(StringBuffer sb, Outlink[] o) {
+    for (int i = 0; i < o.length; i++) {
+      sb.append(o[i].toString());
+      sb.append(System.getProperty("line.separator"));
+    }
+  }
+
+  private static final String outlinksString(Outlink[] o) {
+    StringBuffer sb = new StringBuffer();
+    appendOutlinks(sb, o);
+    return sb.toString();
+  }
+
+  private static final void compareOutlinks(int test, Outlink[] o1, Outlink[] o2) {
+    if (o1.length != o2.length) {
+      assertTrue(
+          "test " + test + ", got wrong number of outlinks (expecting "
+              + o1.length + ", got " + o2.length + ")"
+              + System.getProperty("line.separator") + "answer: "
+              + System.getProperty("line.separator") + outlinksString(o1)
+              + System.getProperty("line.separator") + "got: "
+              + System.getProperty("line.separator") + outlinksString(o2)
+              + System.getProperty("line.separator"), false);
+    }
+
+    for (int i = 0; i < o1.length; i++) {
+      if (!o1[i].equals(o2[i])) {
+        assertTrue(
+            "test " + test + ", got wrong outlinks at position " + i
+                + System.getProperty("line.separator") + "answer: "
+                + System.getProperty("line.separator") + o1[i].toString()
+                + System.getProperty("line.separator") + "got: "
+                + System.getProperty("line.separator") + o2[i].toString(),
+            false);
+
+      }
+    }
+  }
+}