Index: tika-core/src/main/java/org/apache/tika/metadata/Message.java =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/Message.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/metadata/Message.java (revision 0) @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.metadata; + +/** + * A collection of Message related property names. + */ +public interface Message { + String MESSAGE_RECIPIENT_ADDRESS = "Message-Recipient-Address"; + + String MESSAGE_FROM = "Message-From"; + + String MESSAGE_TO = "Message-To"; + + String MESSAGE_CC = "Message-Cc"; + + String MESSAGE_BCC = "Message-Bcc"; +} Index: tika-core/src/main/java/org/apache/tika/metadata/Metadata.java =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/Metadata.java (revision 955241) +++ tika-core/src/main/java/org/apache/tika/metadata/Metadata.java (working copy) @@ -25,7 +25,7 @@ * A multi-valued metadata container. */ public class Metadata implements CreativeCommons, DublinCore, HttpHeaders, - MSOffice, ClimateForcast, TikaMetadataKeys, TikaMimeKeys { + Message, MSOffice, ClimateForcast, TikaMetadataKeys, TikaMimeKeys { /** * A map of all metadata attributes. Index: tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java (revision 955241) +++ tika-parsers/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java (working copy) @@ -16,14 +16,12 @@ */ package org.apache.tika.parser.microsoft; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; -import java.util.Map; -import org.apache.poi.hsmf.datatypes.Chunks; -import org.apache.poi.hsmf.datatypes.StringChunk; +import org.apache.poi.hsmf.MAPIMessage; +import org.apache.poi.hsmf.datatypes.AttachmentChunks; import org.apache.poi.hsmf.exceptions.ChunkNotFoundException; -import org.apache.poi.hsmf.parsers.POIFSChunkParser; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; @@ -38,67 +36,115 @@ /** * Outlook Message Parser. */ -class OutlookExtractor { - - private final Chunks chunks; - - private final POIFSChunkParser parser; - +public class OutlookExtractor { + private final MAPIMessage msg; private final ParseContext context; - public OutlookExtractor( - POIFSFileSystem filesystem, ParseContext context) - throws TikaException { + public OutlookExtractor(POIFSFileSystem filesystem, ParseContext context) throws TikaException { try { + this.msg = new MAPIMessage(filesystem); this.context = context; - this.parser = new POIFSChunkParser(filesystem); - this.chunks = parser.identifyChunks(); } catch (IOException e) { - throw new TikaException("Failed to parse Outlook chunks", e); + throw new TikaException("Failed to parse Outlook message", e); } } - @SuppressWarnings("unchecked") public void parse(XHTMLContentHandler xhtml, Metadata metadata) throws TikaException, SAXException { - String subject = getChunk(chunks.subjectChunk); - String from = getChunk(chunks.displayFromChunk); - - metadata.set(Metadata.AUTHOR, from); - metadata.set(Metadata.TITLE, subject); - metadata.set(Metadata.SUBJECT, getChunk(chunks.conversationTopic)); - - xhtml.element("h1", subject); - - xhtml.startElement("dl"); - header(xhtml, "From", from); - header(xhtml, "To", getChunk(chunks.displayToChunk)); - header(xhtml, "Cc", getChunk(chunks.displayCCChunk)); - header(xhtml, "Bcc", getChunk(chunks.displayBCCChunk)); - xhtml.endElement("dl"); - - xhtml.element("p", getChunk(chunks.textBodyChunk)); - - Map attachments = parser.getAttachmentList(); - for (String key : attachments.keySet()) - { - xhtml.startElement("div", "class", "attachment-entry"); - Metadata entrydata = new Metadata(); - if (key != null && key.length() > 0) { - entrydata.set(Metadata.RESOURCE_NAME_KEY, key); - xhtml.element("h1", key); - } - try { - // Use the delegate parser to parse this entry - context.get(Parser.class, EmptyParser.INSTANCE).parse( - attachments.get(key), - new EmbeddedContentHandler(new BodyContentHandler(xhtml)), - entrydata, - context); - } catch (Exception e) { - // Could not parse the entry, just skip the content - } - xhtml.endElement("div"); + try { + msg.setReturnNullOnMissingChunk(true); + + String subject = msg.getSubject(); + String from = msg.getDisplayFrom(); + + metadata.set(Metadata.AUTHOR, from); + metadata.set(Metadata.MESSAGE_FROM, from); + metadata.set(Metadata.MESSAGE_TO, msg.getDisplayTo()); + metadata.set(Metadata.MESSAGE_CC, msg.getDisplayCC()); + metadata.set(Metadata.MESSAGE_BCC, msg.getDisplayBCC()); + + metadata.set(Metadata.TITLE, subject); + metadata.set(Metadata.SUBJECT, msg.getConversationTopic()); + + try { + for(String recipientAddress : msg.getRecipientEmailAddressList()) { + if(recipientAddress != null) + metadata.add(Metadata.MESSAGE_RECIPIENT_ADDRESS, recipientAddress); + } + } catch(ChunkNotFoundException he) {} // Will be fixed in POI 3.7 Final + + // Date - try two ways to find it + // First try via the proper chunk + if(msg.getMessageDate() != null) { + metadata.set(Metadata.EDIT_TIME, msg.getMessageDate().getTime().toString()); + metadata.set(Metadata.LAST_SAVED, msg.getMessageDate().getTime().toString()); + } else { + try { + // Failing that try via the raw headers + String[] headers = msg.getHeaders(); + if(headers != null && headers.length > 0) { + for(String header: headers) { + if(header.toLowerCase().startsWith("date:")) { + String date = header.substring(header.indexOf(':')+1); + metadata.set(Metadata.EDIT_TIME, date); + metadata.set(Metadata.LAST_SAVED, date); + break; + } + } + } + } catch(ChunkNotFoundException he) { + // We can't find the date, sorry... + } + } + + + xhtml.element("h1", subject); + + // Output the from and to details in text, as you + // often want them in text form for searching + xhtml.startElement("dl"); + header(xhtml, "From", from); + header(xhtml, "To", msg.getDisplayTo()); + header(xhtml, "Cc", msg.getDisplayCC()); + header(xhtml, "Bcc", msg.getDisplayBCC()); + try { + header(xhtml, "Recipients", msg.getRecipientEmailAddress()); + } catch(ChunkNotFoundException e) {} + xhtml.endElement("dl"); + + xhtml.element("p", msg.getTextBody()); + + for (AttachmentChunks attachment : msg.getAttachmentFiles()) { + xhtml.startElement("div", "class", "attachment-entry"); + Metadata entrydata = new Metadata(); + + String filename = null; + if (attachment.attachLongFileName != null) { + filename = attachment.attachLongFileName.getValue(); + } else if (attachment.attachFileName != null) { + filename = attachment.attachFileName.getValue(); + } + if (filename != null && filename.length() > 0) { + entrydata.set(Metadata.RESOURCE_NAME_KEY, filename); + xhtml.element("h1", filename); + } + + try { + // Use the delegate parser to parse this entry + context.get(Parser.class, EmptyParser.INSTANCE).parse( + new ByteArrayInputStream(attachment.attachData.getValue()), + new EmbeddedContentHandler(new BodyContentHandler(xhtml)), + entrydata, + context + ); + } catch (Exception e) { + // Could not parse the entry, just skip the content + } + xhtml.endElement("div"); + + } + } catch(ChunkNotFoundException e) { + throw new TikaException("POI MAPIMessage broken - didn't return null on missing chunk", e); } } @@ -109,21 +155,4 @@ xhtml.element("dd", value); } } - - /** - * Returns the content of the identified string chunk in the - * current document. Returns the empty string if the identified - * chunk does not exist in the current document. - * - * @param chunk string chunk identifier - * @return content of the identified chunk, or the empty string - */ - private String getChunk(StringChunk chunk) { - try { - return parser.getDocumentNode(chunk).toString(); - } catch (ChunkNotFoundException e) { - return ""; - } - } - } Index: tika-parsers/src/main/java/org/apache/tika/parser/mbox/MboxParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/mbox/MboxParser.java (revision 955241) +++ tika-parsers/src/main/java/org/apache/tika/parser/mbox/MboxParser.java (working copy) @@ -47,6 +47,7 @@ public static final String MBOX_MIME_TYPE = "application/mbox"; public static final String MBOX_RECORD_DIVIDER = "From "; private static final Pattern EMAIL_HEADER_PATTERN = Pattern.compile("([^ ]+):[ \t]*(.*)"); + private static final Pattern EMAIL_ADDRESS_PATTERN = Pattern.compile("<(.*@.*)>"); private static final String EMAIL_HEADER_METADATA_PREFIX = MboxParser.class.getSimpleName() + "-"; private static final String EMAIL_FROMLINE_METADATA = EMAIL_HEADER_METADATA_PREFIX + "from"; @@ -191,6 +192,23 @@ if (headerTag.equalsIgnoreCase("From")) { metadata.add(Metadata.AUTHOR, headerContent); metadata.add(Metadata.CREATOR, headerContent); + } else if (headerTag.equalsIgnoreCase("To") || + headerTag.equalsIgnoreCase("Cc") || + headerTag.equalsIgnoreCase("Bcc")) { + Matcher address = EMAIL_ADDRESS_PATTERN.matcher(headerContent); + if(address.find()) { + metadata.add(Metadata.MESSAGE_RECIPIENT_ADDRESS, address.group(1)); + } else if(headerContent.indexOf('@') > -1) { + metadata.add(Metadata.MESSAGE_RECIPIENT_ADDRESS, headerContent); + } + + String property = Metadata.MESSAGE_TO; + if (headerTag.equalsIgnoreCase("Cc")) { + property = Metadata.MESSAGE_CC; + } else if (headerTag.equalsIgnoreCase("Bcc")) { + property = Metadata.MESSAGE_BCC; + } + metadata.add(property, headerContent); } else if (headerTag.equalsIgnoreCase("Subject")) { metadata.add(Metadata.SUBJECT, headerContent); metadata.add(Metadata.TITLE, headerContent); Index: tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java (revision 955241) +++ tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java (working copy) @@ -19,12 +19,10 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.times; import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; import junit.framework.TestCase; @@ -32,7 +30,6 @@ import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; import org.apache.tika.sax.XHTMLContentHandler; - import org.xml.sax.Attributes; import org.xml.sax.ContentHandler; import org.xml.sax.helpers.DefaultHandler; @@ -76,6 +73,7 @@ assertEquals("subject", metadata.get(Metadata.SUBJECT)); assertEquals("", metadata.get(Metadata.AUTHOR)); assertEquals("", metadata.get(Metadata.CREATOR)); + assertEquals(null, metadata.get(Metadata.MESSAGE_RECIPIENT_ADDRESS)); assertEquals("", metadata.get("MboxParser-return-path")); assertEquals("Tue, 9 Jun 2009 23:58:45 -0400", metadata.get(Metadata.DATE)); } catch (Exception e) { @@ -134,6 +132,12 @@ try { parser.parse(stream, handler, metadata, new ParseContext()); + assertEquals("Re: question about when shuffle/sort start working", metadata.get(Metadata.TITLE)); + assertEquals("Re: question about when shuffle/sort start working", metadata.get(Metadata.SUBJECT)); + assertEquals("Jothi Padmanabhan ", metadata.get(Metadata.AUTHOR)); + assertEquals("Jothi Padmanabhan ", metadata.get(Metadata.CREATOR)); + assertEquals("core-user@hadoop.apache.org", metadata.get(Metadata.MESSAGE_RECIPIENT_ADDRESS)); + verify(handler).startDocument(); verify(handler, times(3)).startElement(eq(XHTMLContentHandler.XHTML), eq("p"), eq("p"), any(Attributes.class)); verify(handler, times(3)).endElement(eq(XHTMLContentHandler.XHTML), eq("p"), eq("p")); Index: tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java (revision 955241) +++ tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java (working copy) @@ -52,14 +52,17 @@ assertEquals( "Microsoft Outlook Express 6", metadata.get(Metadata.TITLE)); - // TODO: There's apparently some encoding issue in POI - //assertEquals( - // "L'\u00C9quipe Microsoft Outlook Express", - // metadata.get(Metadata.AUTHOR)); + assertEquals( + "Nouvel utilisateur de Outlook Express", + metadata.get(Metadata.MESSAGE_RECIPIENT_ADDRESS)); + assertEquals( + "L'\u00C9quipe Microsoft Outlook Express", + metadata.get(Metadata.AUTHOR)); String content = handler.toString(); + assertTrue(content.contains("")); assertTrue(content.contains("Microsoft Outlook Express 6")); - //assertTrue(content.contains("L'\u00C9quipe Microsoft Outlook Express")); + assertTrue(content.contains("L'\u00C9quipe Microsoft Outlook Express")); assertTrue(content.contains("Nouvel utilisateur de Outlook Express")); assertTrue(content.contains("Messagerie et groupes de discussion")); }