Index: tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java (revision 1437969) +++ tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java (working copy) @@ -54,7 +54,23 @@ * Parser for ODF content.xml files. */ public class OpenDocumentContentParser extends AbstractParser { + private interface Style { + } + private static class TextStyle implements Style { + public boolean italic; + public boolean bold; + public boolean underlined; + } + + private static class ListStyle implements Style { + public boolean ordered; + + public String getTag() { + return ordered ? "ol" : "ul"; + } + } + private static final class OpenDocumentElementMappingContentHandler extends ElementMappingContentHandler { private final ContentHandler handler; @@ -62,6 +78,12 @@ private int nodeDepth = 0; private int completelyFiltered = 0; private Stack headingStack = new Stack(); + private Map textStyleMap = new HashMap(); + private Map listStyleMap = new HashMap(); + private TextStyle textStyle; + private TextStyle lastTextStyle; + private Stack listStyleStack = new Stack(); + private ListStyle listStyle; private OpenDocumentElementMappingContentHandler(ContentHandler handler, Map mappings) { @@ -75,6 +97,7 @@ // only forward content of tags from text:-namespace if (completelyFiltered == 0 && nodeDepth > 0 && textNodeStack.get(nodeDepth - 1)) { + lazyEndSpan(); super.characters(ch,start,length); } } @@ -124,10 +147,85 @@ return false; } + private void startList(String name) throws SAXException { + String elementName = "ul"; + if (name != null) { + ListStyle style = listStyleMap.get(name); + elementName = style != null ? style.getTag() : "ul"; + listStyleStack.push(style); + } + handler.startElement(XHTML, elementName, elementName, EMPTY_ATTRIBUTES); + } + + private void endList() throws SAXException { + String elementName = "ul"; + if (!listStyleStack.isEmpty()) { + ListStyle style = listStyleStack.pop(); + elementName = style != null ? style.getTag() : "ul"; + } + handler.endElement(XHTML, elementName, elementName); + } + + private void startSpan(String name) throws SAXException { + if (name == null) { + return; + } + + TextStyle style = textStyleMap.get(name); + + // End tags that refer to no longer valid styles + if (!style.underlined && lastTextStyle != null && lastTextStyle.underlined) { + handler.endElement(XHTML, "u", "u"); + } + if (!style.italic && lastTextStyle != null && lastTextStyle.italic) { + handler.endElement(XHTML, "i", "i"); + } + if (!style.bold && lastTextStyle != null && lastTextStyle.bold) { + handler.endElement(XHTML, "b", "b"); + } + + // Start tags for new styles + if (style.bold && (lastTextStyle == null || !lastTextStyle.bold)) { + handler.startElement(XHTML, "b", "b", EMPTY_ATTRIBUTES); + } + if (style.italic && (lastTextStyle == null || !lastTextStyle.italic)) { + handler.startElement(XHTML, "i", "i", EMPTY_ATTRIBUTES); + } + if (style.underlined && (lastTextStyle == null || !lastTextStyle.underlined)) { + handler.startElement(XHTML, "u", "u", EMPTY_ATTRIBUTES); + } + + textStyle = style; + lastTextStyle = null; + } + + private void endSpan() throws SAXException { + lastTextStyle = textStyle; + textStyle = null; + } + + private void lazyEndSpan() throws SAXException { + if (lastTextStyle == null) { + return; + } + + if (lastTextStyle.underlined) { + handler.endElement(XHTML, "u", "u"); + } + if (lastTextStyle.italic) { + handler.endElement(XHTML, "i", "i"); + } + if (lastTextStyle.bold) { + handler.endElement(XHTML, "b", "b"); + } + + lastTextStyle = null; + } + @Override public void startElement( String namespaceURI, String localName, String qName, - Attributes atts) throws SAXException { + Attributes attrs) throws SAXException { // keep track of current node type. If it is a text node, // a bit at the current depth ist set in textNodeStack. // characters() checks the top bit to determine, if the @@ -135,6 +233,42 @@ // the depth of the current node and also marks top of stack. assert nodeDepth >= 0; + // Set styles + if (STYLE_NS.equals(namespaceURI) && "style".equals(localName)) { + String family = attrs.getValue(STYLE_NS, "family"); + if ("text".equals(family)) { + textStyle = new TextStyle(); + String name = attrs.getValue(STYLE_NS, "name"); + textStyleMap.put(name, textStyle); + } + } else if (TEXT_NS.equals(namespaceURI) && "list-style".equals(localName)) { + listStyle = new ListStyle(); + String name = attrs.getValue(STYLE_NS, "name"); + listStyleMap.put(name, listStyle); + } else if (textStyle != null && STYLE_NS.equals(namespaceURI) + && "text-properties".equals(localName)) { + String fontStyle = attrs.getValue(FORMATTING_OBJECTS_NS, "font-style"); + if ("italic".equals(fontStyle) || "oblique".equals(fontStyle)) { + textStyle.italic = true; + } + String fontWeight = attrs.getValue(FORMATTING_OBJECTS_NS, "font-weight"); + if ("bold".equals(fontWeight) || "bolder".equals(fontWeight) + || (fontWeight!=null && Character.isDigit(fontWeight.charAt(0)) + && Integer.valueOf(fontWeight) > 500)) { + textStyle.bold = true; + } + String underlineStyle = attrs.getValue(STYLE_NS, "text-underline-style"); + if (underlineStyle != null) { + textStyle.underlined = true; + } + } else if (listStyle != null && TEXT_NS.equals(namespaceURI)) { + if ("list-level-style-bullet".equals(localName)) { + listStyle.ordered = false; + } else if ("list-level-style-number".equals(localName)) { + listStyle.ordered = true; + } + } + textNodeStack.set(nodeDepth++, isTextNode(namespaceURI, localName)); // filter *all* content of some tags @@ -148,11 +282,14 @@ // special handling of text:h, that are directly passed // to incoming handler if (TEXT_NS.equals(namespaceURI) && "h".equals(localName)) { - final String el = headingStack.push(getXHTMLHeaderTagName(atts)); + final String el = headingStack.push(getXHTMLHeaderTagName(attrs)); handler.startElement(XHTMLContentHandler.XHTML, el, el, EMPTY_ATTRIBUTES); + } else if (TEXT_NS.equals(namespaceURI) && "list".equals(localName)) { + startList(attrs.getValue(TEXT_NS, "style-name")); + } else if (TEXT_NS.equals(namespaceURI) && "span".equals(localName)) { + startSpan(attrs.getValue(TEXT_NS, "style-name")); } else { - super.startElement( - namespaceURI, localName, qName, atts); + super.startElement(namespaceURI, localName, qName, attrs); } } } @@ -161,6 +298,12 @@ public void endElement( String namespaceURI, String localName, String qName) throws SAXException { + if (STYLE_NS.equals(namespaceURI) && "style".equals(localName)) { + textStyle = null; + } else if (TEXT_NS.equals(namespaceURI) && "list-style".equals(localName)) { + listStyle = null; + } + // call next handler if no filtering if (completelyFiltered == 0) { // special handling of text:h, that are directly passed @@ -168,7 +311,14 @@ if (TEXT_NS.equals(namespaceURI) && "h".equals(localName)) { final String el = headingStack.pop(); handler.endElement(XHTMLContentHandler.XHTML, el, el); + } else if (TEXT_NS.equals(namespaceURI) && "list".equals(localName)) { + endList(); + } else if (TEXT_NS.equals(namespaceURI) && "span".equals(localName)) { + endSpan(); } else { + if (TEXT_NS.equals(namespaceURI) && "p".equals(localName)) { + lazyEndSpan(); + } super.endElement(namespaceURI,localName,qName); } @@ -208,6 +358,12 @@ public static final String TABLE_NS = "urn:oasis:names:tc:opendocument:xmlns:table:1.0"; + public static final String STYLE_NS = + "urn:oasis:names:tc:opendocument:xmlns:style:1.0"; + + public static final String FORMATTING_OBJECTS_NS = + "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"; + public static final String OFFICE_NS = "urn:oasis:names:tc:opendocument:xmlns:office:1.0"; @@ -244,9 +400,6 @@ new QName(TEXT_NS, "line-break"), new TargetElement(XHTML, "br")); MAPPINGS.put( - new QName(TEXT_NS, "list"), - new TargetElement(XHTML, "ul")); - MAPPINGS.put( new QName(TEXT_NS, "list-item"), new TargetElement(XHTML, "li")); MAPPINGS.put( Index: tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java (revision 1437969) +++ tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java (working copy) @@ -16,6 +16,7 @@ */ package org.apache.tika.parser.odf; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -23,12 +24,12 @@ import java.util.HashSet; import java.util.Set; import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; -//import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -//import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.tika.exception.TikaException; import org.apache.tika.io.IOUtils; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.AbstractParser; @@ -129,12 +130,9 @@ } */ - // TODO: if incoming IS is a TIS with a file - // associated, we should open ZipFile so we can - // visit metadata, mimetype first; today we lose - // all the metadata if meta.xml is hit after - // content.xml in the stream. Then we can still - // read-once for the content.xml. + // If incoming InputStream is a TikaInputStream with a file + // associated, we open ZipFile so we can visit meta.xml, mimetype, and styles.xml first + // so that we do not lose all the metadata. XHTMLContentHandler xhtml = new XHTMLContentHandler(baseHandler, metadata); @@ -146,34 +144,62 @@ // Process the file in turn ZipInputStream zip = new ZipInputStream(stream); ZipEntry entry = zip.getNextEntry(); + boolean parsedMeta = false; + boolean parsedStyles = false; + File file = null; + while (entry != null) { if (entry.getName().equals("mimetype")) { String type = IOUtils.toString(zip, "UTF-8"); metadata.set(Metadata.CONTENT_TYPE, type); } else if (entry.getName().equals("meta.xml")) { meta.parse(zip, new DefaultHandler(), metadata, context); + parsedMeta = true; } else if (entry.getName().endsWith("content.xml")) { - if (content instanceof OpenDocumentContentParser) { - ((OpenDocumentContentParser) content).parseInternal(zip, handler, metadata, context); - } else { - // Foreign content parser was set: - content.parse(zip, handler, metadata, context); + if (!parsedMeta && !parsedStyles && stream instanceof TikaInputStream) { + TikaInputStream tikaStream = (TikaInputStream)stream; + if (tikaStream.hasFile()) { + file = tikaStream.getFile(); + } } + if (file == null) { + parseContent(zip, handler, metadata, context); + } } else if (entry.getName().endsWith("styles.xml")) { - if (content instanceof OpenDocumentContentParser) { - ((OpenDocumentContentParser) content).parseInternal(zip, handler, metadata, context); - } else { - // Foreign content parser was set: - content.parse(zip, handler, metadata, context); - } + parseContent(zip, handler, metadata, context); + parsedStyles = true; } entry = zip.getNextEntry(); } + if (file != null) { + // contents.xml has not been parsed yet, parse it now + ZipFile zipFile = new ZipFile(file); + try + { + ZipEntry zipEntry = zipFile.getEntry("content.xml"); + InputStream zipStream = zipFile.getInputStream(zipEntry); + parseContent(zipStream, handler, metadata, context); + } + finally + { + zipFile.close(); + } + } + // Only now call the end document - if(handler.getEndDocumentWasCalled()) { - handler.reallyEndDocument(); + if (handler.getEndDocumentWasCalled()) { + handler.reallyEndDocument(); } } + protected void parseContent(InputStream stream, ContentHandler handler, Metadata metadata, + ParseContext context) throws IOException, SAXException, TikaException { + if (content instanceof OpenDocumentContentParser) { + ((OpenDocumentContentParser)content).parseInternal(stream, handler, metadata, context); + } else { + // Foreign content parser was set: + content.parse(stream, handler, metadata, context); + } + } } Index: tika-parsers/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java (revision 1437969) +++ tika-parsers/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java (working copy) @@ -44,8 +44,7 @@ public void testOO3() throws Exception { for (Parser parser : getParsers()) { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testODFwithOOo3.odt"); + InputStream input = getResourceAsStream("/test-documents/testODFwithOOo3.odt"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -69,8 +68,7 @@ public void testOO2() throws Exception { for (Parser parser : getParsers()) { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testOpenOffice2.odt"); + InputStream input = getResourceAsStream("/test-documents/testOpenOffice2.odt"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -140,8 +138,7 @@ * OO2 file with different metadata in it */ public void testOO2Metadata() throws Exception { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testOpenOffice2.odf"); + InputStream input = getResourceAsStream("/test-documents/testOpenOffice2.odf"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -201,8 +198,7 @@ * Similar to {@link #testXMLParser()}, but using an OO3 file */ public void testOO3Metadata() throws Exception { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testODFwithOOo3.odt"); + InputStream input = getResourceAsStream("/test-documents/testODFwithOOo3.odt"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -270,8 +266,7 @@ } public void testODPMasterFooter() throws Exception { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testMasterFooter.odp"); + InputStream input = getResourceAsStream("/test-documents/testMasterFooter.odp"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -285,8 +280,7 @@ } public void testODTFooter() throws Exception { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testFooter.odt"); + InputStream input = getResourceAsStream("/test-documents/testFooter.odt"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -302,8 +296,7 @@ } public void testODSFooter() throws Exception { - InputStream input = ODFParserTest.class.getResourceAsStream( - "/test-documents/testFooter.ods"); + InputStream input = getResourceAsStream("/test-documents/testFooter.ods"); try { Metadata metadata = new Metadata(); ContentHandler handler = new BodyContentHandler(); @@ -314,5 +307,16 @@ } finally { input.close(); } - } + } + + public void testODTStyles() throws Exception { + String xml = getXML("testStyles.odt").xml; + System.out.println(xml); + assertContains("This is just a test", xml); + assertContains("

And another test is here.

", xml); + assertContains("
    \t

  1. One

    ", xml); + assertContains("
", xml); + assertContains("
    \t

  • First

    ", xml); + assertContains("
", xml); + } }