Index: tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java (revision 965379) +++ tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java (working copy) @@ -180,7 +180,7 @@ // Get the HTML mapper from the parse context HtmlMapper mapper = - context.get(HtmlMapper.class, new HtmlParserMapper()); + context.get(HtmlMapper.class, new HtmlParserMapper(metadata,context)); // Parse the HTML document org.ccil.cowan.tagsoup.Parser parser = @@ -253,7 +253,10 @@ * @deprecated Use the {@link HtmlMapper} mechanism to customize * the HTML mapping. This class will be removed in Tika 1.0. */ - private class HtmlParserMapper implements HtmlMapper { + private class HtmlParserMapper extends HtmlMapper { + HtmlParserMapper(Metadata metadata, ParseContext context) { + super(metadata, context); + } public String mapSafeElement(String name) { return HtmlParser.this.mapSafeElement(name); } Index: tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java (revision 965379) +++ tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java (working copy) @@ -37,6 +37,8 @@ private final Metadata metadata; + private boolean inHead = false; + private int bodyLevel = 0; private int discardLevel = 0; @@ -45,8 +47,8 @@ private final StringBuilder title = new StringBuilder(); - private HtmlHandler( - HtmlMapper mapper, XHTMLContentHandler xhtml, Metadata metadata) { + private HtmlHandler(HtmlMapper mapper, XHTMLContentHandler xhtml, + Metadata metadata) { super(xhtml); this.mapper = mapper; this.xhtml = xhtml; @@ -67,15 +69,17 @@ } } - public HtmlHandler( - HtmlMapper mapper, ContentHandler handler, Metadata metadata) { + public HtmlHandler(HtmlMapper mapper, ContentHandler handler, + Metadata metadata) { this(mapper, new XHTMLContentHandler(handler, metadata), metadata); } @Override - public void startElement( - String uri, String local, String name, Attributes atts) - throws SAXException { + public void startElement(String uri, String local, String name, + Attributes atts) throws SAXException { + if ("HEAD".equals(name)) { + inHead = true; + } if ("TITLE".equals(name) || titleLevel > 0) { titleLevel++; } @@ -86,25 +90,25 @@ discardLevel++; } - if (bodyLevel == 0 && discardLevel == 0) { + // special cases for elements in header + if (inHead && discardLevel == 0) { if ("META".equals(name) && atts.getValue("content") != null) { if (atts.getValue("http-equiv") != null) { - metadata.set( - atts.getValue("http-equiv"), + metadata.set(atts.getValue("http-equiv"), atts.getValue("content")); xhtml.startElement(uri, local, "meta", atts); } if (atts.getValue("name") != null) { // Record the meta tag in the metadata - metadata.set( - atts.getValue("name"), + metadata.set(atts.getValue("name"), atts.getValue("content")); // Normalise if possible - if(atts.getValue("name").equalsIgnoreCase("ICBM")) { - Matcher m = Pattern.compile( - "\\s*(-?\\d+\\.\\d+)[,\\s]+(-?\\d+\\.\\d+)\\s*" - ).matcher(atts.getValue("content")); - if(m.matches()) { + if (atts.getValue("name").equalsIgnoreCase("ICBM")) { + Matcher m = Pattern + .compile( + "\\s*(-?\\d+\\.\\d+)[,\\s]+(-?\\d+\\.\\d+)\\s*") + .matcher(atts.getValue("content")); + if (m.matches()) { metadata.set(Metadata.LATITUDE, m.group(1)); metadata.set(Metadata.LONGITUDE, m.group(2)); } @@ -113,35 +117,21 @@ xhtml.startElement(uri, local, "meta", atts); } } else if ("BASE".equals(name) && atts.getValue("href") != null) { - metadata.set( - Metadata.CONTENT_LOCATION, - resolve(atts.getValue("href").trim())); + metadata.set(Metadata.CONTENT_LOCATION, + mapper.resolve(atts.getValue("href").trim())); xhtml.startElement(uri, local, "base", atts); } else if ("LINK".equals(name) && atts.getValue("href") != null) { xhtml.startElement(uri, local, "link", atts); } } - if (bodyLevel > 0 && discardLevel == 0) { + if (discardLevel == 0) { String safe = mapper.mapSafeElement(name); - if (safe != null) { - // check if there are any attributes to process - if (atts.getLength()==0) xhtml.startElement(safe); - else { - AttributesImpl newAttributes = new AttributesImpl(atts); - for (int att=0;att 0 && discardLevel == 0) { + } else if (discardLevel == 0) { String safe = mapper.mapSafeElement(name); if (safe != null) { xhtml.endElement(safe); } else if ("A".equals(name)) { xhtml.endElement("a"); - } else if (XHTMLContentHandler.ENDLINE.contains( - name.toLowerCase())) { + } else if (XHTMLContentHandler.ENDLINE.contains(name.toLowerCase())) { // TIKA-343: Replace closing block tags (and
) with a // newline unless the HtmlMapper above has already mapped // them to something else @@ -216,42 +223,4 @@ } } - private String resolve(String url) { - // Return the URL as-is if no base URL is available - if (metadata.get(Metadata.CONTENT_LOCATION) == null) { - return url; - } - - // Check for common non-hierarchical and pseudo URI prefixes - String lower = url.toLowerCase(); - if (lower.startsWith("urn:") - || lower.startsWith("mailto:") - || lower.startsWith("tel:") - || lower.startsWith("data:") - || lower.startsWith("javascript:") - || lower.startsWith("about:")) { - return url; - } - - try { - URL base = new URL(metadata.get(Metadata.CONTENT_LOCATION).trim()); - - // We need to handle one special case, where the relativeUrl is - // just a query string (like "?pid=1"), and the baseUrl doesn't - // end with a '/'. In that case, the URL class removes the last - // portion of the path, which we don't want. - String path = base.getPath(); - if (url.startsWith("?") && path.length() > 0 && !path.endsWith("/")) { - return new URL( - base.getProtocol(), base.getHost(), base.getPort(), - base.getPath() + url).toExternalForm(); - } else { - return new URL(base, url).toExternalForm(); - } - } catch (MalformedURLException e) { - // Unknown or broken format; just return the URL as received. - return url; - } - } - } \ No newline at end of file Index: tika-parsers/src/main/java/org/apache/tika/parser/html/IdentityHtmlMapper.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/html/IdentityHtmlMapper.java (revision 965379) +++ tika-parsers/src/main/java/org/apache/tika/parser/html/IdentityHtmlMapper.java (working copy) @@ -16,16 +16,23 @@ */ package org.apache.tika.parser.html; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + /** * Alternative HTML mapping rules that pass the input HTML as-is without any * modifications. * * @since Apache Tika 0.8 */ -public class IdentityHtmlMapper implements HtmlMapper { +public class IdentityHtmlMapper extends HtmlMapper { - public static final HtmlMapper INSTANCE = new IdentityHtmlMapper(); + IdentityHtmlMapper(Metadata metadata, ParseContext context) { + super(metadata, context); + } + public static final HtmlMapper INSTANCE = new IdentityHtmlMapper(new Metadata(),new ParseContext()); + public boolean isDiscardElement(String name) { return false; } Index: tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlMapper.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlMapper.java (revision 965379) +++ tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlMapper.java (working copy) @@ -16,54 +16,109 @@ */ package org.apache.tika.parser.html; +import java.net.MalformedURLException; +import java.net.URL; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + /** - * HTML mapper used to make incoming HTML documents easier to handle by - * Tika clients. The {@link HtmlParser} looks up an optional HTML mapper from - * the parse context and uses it to map parsed HTML to "safe" XHTML. A client - * that wants to customize this mapping can place a custom HtmlMapper instance - * into the parse context. - * + * HTML mapper used to make incoming HTML documents easier to handle by Tika + * clients. The {@link HtmlParser} looks up an optional HTML mapper from the + * parse context and uses it to map parsed HTML to "safe" XHTML. A client that + * wants to customize this mapping can place a custom HtmlMapper instance into + * the parse context. + * * @since Apache Tika 0.6 */ -public interface HtmlMapper { +public abstract class HtmlMapper { + Metadata metadata; + ParseContext context; + + HtmlMapper(Metadata metadata, ParseContext context) { + this.metadata = metadata; + this.context = context; + } + /** * Maps "safe" HTML element names to semantic XHTML equivalents. If the * given element is unknown or deemed unsafe for inclusion in the parse - * output, then this method returns null and the element - * will be ignored but the content inside it is still processed. See - * the {@link #isDiscardElement(String)} method for a way to discard - * the entire contents of an element. - * - * @param name HTML element name (upper case) - * @return XHTML element name (lower case), or - * null if the element is unsafe + * output, then this method returns null and the element will + * be ignored but the content inside it is still processed. See the + * {@link #isDiscardElement(String)} method for a way to discard the entire + * contents of an element. + * + * @param name + * HTML element name (upper case) + * @return XHTML element name (lower case), or null if the + * element is unsafe */ - String mapSafeElement(String name); + abstract String mapSafeElement(String name); /** * Checks whether all content within the given HTML element should be * discarded instead of including it in the parse output. - * - * @param name HTML element name (upper case) - * @return true if content inside the named element - * should be ignored, false otherwise + * + * @param name + * HTML element name (upper case) + * @return true if content inside the named element should be + * ignored, false otherwise */ - boolean isDiscardElement(String name); - - + abstract boolean isDiscardElement(String name); + /** * Maps "safe" HTML attribute names to semantic XHTML equivalents. If the * given attribute is unknown or deemed unsafe for inclusion in the parse - * output, then this method returns null and the attribute - * will be ignored. This method assumes that the element name - * is valid and normalised. - * - * @param elementName HTML element name (lower case) - * @param attributeName HTML attribute name (lower case) - * @return XHTML attribute name (lower case), or - * null if the element is unsafe + * output, then this method returns null and the attribute will + * be ignored. This method assumes that the element name is valid and + * normalised. + * + * @param elementName + * HTML element name (lower case) + * @param attributeName + * HTML attribute name (lower case) + * @return XHTML attribute name (lower case), or null if the + * element is unsafe */ - String mapSafeAttribute(String elementName, String attributeName); + abstract String mapSafeAttribute(String elementName, String attributeName); + + /** + * Normalises a URL given a base URL stored in the metadata + **/ + String resolve(String url) { + // Return the URL as-is if no base URL is available + if (metadata.get(Metadata.CONTENT_LOCATION) == null) { + return url; + } + + // Check for common non-hierarchical and pseudo URI prefixes + String lower = url.toLowerCase(); + if (lower.startsWith("urn:") || lower.startsWith("mailto:") + || lower.startsWith("tel:") || lower.startsWith("data:") + || lower.startsWith("javascript:") + || lower.startsWith("about:")) { + return url; + } + + try { + URL base = new URL(metadata.get(Metadata.CONTENT_LOCATION).trim()); + + // We need to handle one special case, where the relativeUrl is + // just a query string (like "?pid=1"), and the baseUrl doesn't + // end with a '/'. In that case, the URL class removes the last + // portion of the path, which we don't want. + String path = base.getPath(); + if (url.startsWith("?") && path.length() > 0 && !path.endsWith("/")) { + return new URL(base.getProtocol(), base.getHost(), + base.getPort(), base.getPath() + url).toExternalForm(); + } else { + return new URL(base, url).toExternalForm(); + } + } catch (MalformedURLException e) { + // Unknown or broken format; just return the URL as received. + return url; + } + } } Index: tika-parsers/src/main/java/org/apache/tika/parser/html/LinksHtmlMapper.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/html/LinksHtmlMapper.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/html/LinksHtmlMapper.java (revision 0) @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.html; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +/** + * Extends the default mapper to include elements containing links + **/ +public class LinksHtmlMapper extends DefaultHtmlMapper { + + LinksHtmlMapper(Metadata metadata, ParseContext context) { + super(metadata, context); + } + + public String mapSafeElement(String name) { + String allowedByDefault = super.mapSafeElement(name); + if (allowedByDefault != null) + return allowedByDefault; + + if ("IMG".equals(name)) + return "img"; + if ("MAP".equals(name)) + return "map"; + if ("OBJECT".equals(name)) + return "object"; + if ("FRAME".equals(name)) + return "frame"; + if ("IFRAME".equals(name)) + return "iframe"; + if ("AREA".equals(name)) + return "area"; + if ("LINK".equals(name)) + return "link"; + + return null; + } + + public String mapSafeAttribute(String elementName, String attributeName) { + return attributeName.toLowerCase(); + } + +} Index: tika-parsers/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java (revision 965379) +++ tika-parsers/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java (working copy) @@ -16,17 +16,24 @@ */ package org.apache.tika.parser.html; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + /** * The default HTML mapping rules in Tika. * * @since Apache Tika 0.6 */ -public class DefaultHtmlMapper implements HtmlMapper { +public class DefaultHtmlMapper extends HtmlMapper { + DefaultHtmlMapper(Metadata metadata, ParseContext context) { + super(metadata, context); + } + /** * @since Apache Tika 0.8 */ - public static final HtmlMapper INSTANCE = new DefaultHtmlMapper(); + public static final HtmlMapper INSTANCE = new DefaultHtmlMapper(new Metadata(),new ParseContext()); public String mapSafeElement(String name) { // Based on http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd