Index: src/test/org/apache/nutch/fetcher/TestFetcher.java =================================================================== --- src/test/org/apache/nutch/fetcher/TestFetcher.java (revision 549101) +++ src/test/org/apache/nutch/fetcher/TestFetcher.java (working copy) @@ -78,6 +78,7 @@ addUrl(urls,"pagea.html"); addUrl(urls,"pageb.html"); addUrl(urls,"dup_of_pagea.html"); + addUrl(urls,"nested_spider_trap.html"); CrawlDBTestUtil.generateSeedList(fs, urlPath, urls); Index: src/testresources/fetch-test-site/nested_spider_trap.html =================================================================== --- src/testresources/fetch-test-site/nested_spider_trap.html (revision 0) +++ src/testresources/fetch-test-site/nested_spider_trap.html (revision 0) @@ -0,0 +1,23 @@ + + +nested spider trap + + +Nutch fetcher test page + + + + + +
+ + + + + + + + +
+ + \ No newline at end of file Property changes on: C:\PROGRAMMING\java\eclipse\workspace\NutchTrunk\src\testresources\fetch-test-site\nested_spider_trap.html ___________________________________________________________________ Name: svn:mime-type + text/html Name: svn:keywords + Date Revision Name: svn:eol-style + native Index: src/plugin/parse-htmlsrc/java/org/apache/nutch/parse/html/DOMContentUtils.java =================================================================== --- src/plugin/parse-htmlsrc/java/org/apache/nutch/parse/html/DOMContentUtils.java (revision 549142) +++ src/plugin/parse-htmlsrc/java/org/apache/nutch/parse/html/DOMContentUtils.java (working copy) @@ -21,6 +21,7 @@ import java.net.MalformedURLException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Stack; import org.apache.nutch.parse.Outlink; import org.apache.hadoop.conf.Configuration; @@ -55,7 +56,6 @@ private HashMap linkParams = new HashMap(); private Configuration conf; - public DOMContentUtils(Configuration conf) { setConf(conf); } @@ -344,56 +344,64 @@ */ public void getOutlinks(URL base, ArrayList outlinks, Node node) { + + Stack nodes = new Stack(); + nodes.add(node); - NodeList children = node.getChildNodes(); - int childLen= 0; - if (children != null) - childLen= children.getLength(); + while (nodes.size() > 0) { + + Node curnode = nodes.pop(); + NodeList children = curnode.getChildNodes(); + int childLen = (children != null) ? children.getLength() : 0; + + // put the children node on the stack in first to last order + for (int i = childLen - 1; i >= 0; i--) { + nodes.add(children.item(i)); + } + + if (curnode.getNodeType() == Node.ELEMENT_NODE) { + + String nodeName = curnode.getNodeName().toLowerCase(); + LinkParams params = (LinkParams)linkParams.get(nodeName); + if (params != null) { + if (!shouldThrowAwayLink(curnode, children, childLen, params)) { - if (node.getNodeType() == Node.ELEMENT_NODE) { - String nodeName = node.getNodeName().toLowerCase(); - LinkParams params = (LinkParams)linkParams.get(nodeName); - if (params != null) { - if (!shouldThrowAwayLink(node, children, childLen, params)) { - - StringBuffer linkText = new StringBuffer(); - getText(linkText, node, true); - - NamedNodeMap attrs = node.getAttributes(); - String target = null; - boolean noFollow = false; - boolean post = false; - for (int i= 0; i < attrs.getLength(); i++ ) { - Node attr = attrs.item(i); - String attrName = attr.getNodeName(); - if (params.attrName.equalsIgnoreCase(attrName)) { - target = attr.getNodeValue(); - } else if ("rel".equalsIgnoreCase(attrName) && - "nofollow".equalsIgnoreCase(attr.getNodeValue())) { - noFollow = true; - } else if ("method".equalsIgnoreCase(attrName) && - "post".equalsIgnoreCase(attr.getNodeValue())) { - post = true; + StringBuffer linkText = new StringBuffer(); + getText(linkText, curnode, true); + + NamedNodeMap attrs = curnode.getAttributes(); + String target = null; + boolean noFollow = false; + boolean post = false; + for (int i= 0; i < attrs.getLength(); i++ ) { + Node attr = attrs.item(i); + String attrName = attr.getNodeName(); + if (params.attrName.equalsIgnoreCase(attrName)) { + target = attr.getNodeValue(); + } else if ("rel".equalsIgnoreCase(attrName) && + "nofollow".equalsIgnoreCase(attr.getNodeValue())) { + noFollow = true; + } else if ("method".equalsIgnoreCase(attrName) && + "post".equalsIgnoreCase(attr.getNodeValue())) { + post = true; + } } + if (target != null && !noFollow && !post) + try { + + URL url = (base.toString().indexOf(';') > 0) ? + fixEmbeddedParams(base, target) : new URL(base, target); + outlinks.add(new Outlink(url.toString(), + linkText.toString().trim(), conf)); + } catch (MalformedURLException e) { + // don't care + } } - if (target != null && !noFollow && !post) - try { - - URL url = (base.toString().indexOf(';') > 0) ? - fixEmbeddedParams(base, target) : new URL(base, target); - outlinks.add(new Outlink(url.toString(), - linkText.toString().trim(), conf)); - } catch (MalformedURLException e) { - // don't care - } + // this should not have any children, skip them + if (params.childLen == 0) continue; } - // this should not have any children, skip them - if (params.childLen == 0) return; } } - for ( int i = 0; i < childLen; i++ ) { - getOutlinks(base, outlinks, children.item(i)); - } } }