diff --git tika-parsers/src/main/java/org/apache/tika/parser/rtf/FieldState.java tika-parsers/src/main/java/org/apache/tika/parser/rtf/FieldState.java index 379b8e771..07cb36e49 100644 --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/FieldState.java +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/FieldState.java @@ -1,4 +1,132 @@ package org.apache.tika.parser.rtf; +/* Holds all state associated with the current RTF field. */ public class FieldState { + + public enum FieldInst { + HYPERLINK, FORMCHECKBOX, FORMTEXT, UNKNOWN + } + + public enum FieldGroups { + fldinst, fldrslt, formfield, ffname + } + + public enum FormFieldType { + TEXT(0), CHECKBOX(1), LIST(2); + + int rtfType; + + FormFieldType(int val) { + rtfType = val; + } + + public static FormFieldType fromRtfType(int rtfValue) { + FormFieldType ret = null; + + switch(rtfValue) { + case 0: ret = TEXT; break; + case 1: ret = CHECKBOX; break; + case 2: ret = LIST; break; + } + + return ret; + } + } + + private FieldInst inst; + private FormFieldType type; + private FieldGroups currentGroup; + private boolean bufferText = false; // should the text found inside the group be buffered? + private boolean checked; // checkbox status + private String url; // hyperlink url + private String result; // result text value + private String name; // ffname text value + + public FieldState() { + } + + public FieldInst getInst() { + return inst; + } + + public void setInst(FieldInst inst) { + this.inst = inst; + } + + public FormFieldType getType() { + return type; + } + + public void setType(FormFieldType type) { + this.type = type; + } + + public FieldGroups getCurrentGroup() { + return currentGroup; + } + + public void setCurrentGroup(FieldGroups currentGroup) { + this.currentGroup = currentGroup; + + if (currentGroup != null) { + switch(currentGroup) { + case fldinst: + bufferText = true; + break; + case formfield: + bufferText = true; + break; + case ffname: + bufferText = true; + break; + case fldrslt: + bufferText = true; + break; + default: + bufferText = false; + } + } else { + bufferText = false; + } + } + + public boolean isBufferText() { + return bufferText; + } + + public void setBufferText(boolean bufferText) { + this.bufferText = bufferText; + } + + public boolean isChecked() { + return checked; + } + + public void setChecked(boolean checked) { + this.checked = checked; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getResult() { + return result; + } + + public void setResult(String result) { + this.result = result; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } } diff --git tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java index dfc0956b4..bfe0aaa2a 100644 --- tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java +++ tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java @@ -17,6 +17,15 @@ package org.apache.tika.parser.rtf; +import org.apache.commons.io.IOUtils; +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.metadata.*; +import org.apache.tika.sax.XHTMLContentHandler; +import org.apache.tika.utils.CharsetUtils; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.AttributesImpl; + import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; @@ -27,26 +36,7 @@ import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; -import java.util.Calendar; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Locale; -import java.util.Map; -import java.util.Stack; -import java.util.TimeZone; - -import org.apache.commons.io.IOUtils; -import org.apache.tika.exception.TikaException; -import org.apache.tika.extractor.EmbeddedDocumentUtil; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.metadata.Office; -import org.apache.tika.metadata.OfficeOpenXMLCore; -import org.apache.tika.metadata.OfficeOpenXMLExtended; -import org.apache.tika.metadata.Property; -import org.apache.tika.metadata.TikaCoreProperties; -import org.apache.tika.sax.XHTMLContentHandler; -import org.apache.tika.utils.CharsetUtils; -import org.xml.sax.SAXException; +import java.util.*; /* Tokenizes and performs a "shallow" parse of the RTF * document, just enough to properly decode the text. @@ -296,8 +286,8 @@ final class TextExtractor { // keywords, etc.) inside the info group: private Property nextMetaData; private boolean inParagraph; - // Non-zero if we are processing inside a field destination: - private int fieldState; + // Non-null if we are processing inside a field destination: + private FieldState fieldState; // Non-zero list index private int pendingListEnd; private Map listTable = new HashMap(); @@ -308,7 +298,7 @@ final class TextExtractor { private boolean ignoreListMarkup; // Non-null if we've seen the url for a HYPERLINK but not yet // its text: - private String pendingURL; + //private String pendingURL; // Used to process the sub-groups inside the upr // group: private int uprState = -1; @@ -432,7 +422,7 @@ final class TextExtractor { pushBytes(); } - if (inHeader || fieldState == 1) { + if (inHeader || fieldState != null) { pendingBuffer.append(ch); } else if (groupState.sn == true || groupState.sv == true) { embObjHandler.writeMetadataChar(ch); @@ -746,7 +736,7 @@ final class TextExtractor { final int pos = outputCharBuffer.position(); if (pos > 0) { - if (inHeader || fieldState == 1) { + if (inHeader || (fieldState != null && fieldState.isBufferText())) { pendingBuffer.append(outputArray, 0, pos); } else { lazyStartParagraph(); @@ -765,7 +755,7 @@ final class TextExtractor { final int pos = outputCharBuffer.position(); if (pos > 0) { - if (inHeader || fieldState == 1) { + if (inHeader || (fieldState != null && fieldState.isBufferText())) { pendingBuffer.append(outputArray, 0, pos); } else { lazyStartParagraph(); @@ -1035,6 +1025,17 @@ final class TextExtractor { // log some warning? } } + + // Process field control words + if (fieldState != null) { + if (equals("fftype")) { + fieldState.setType(FieldState.FormFieldType.fromRtfType(param)); + } else if (equals("ffres")) { + if (fieldState.getType() == FieldState.FormFieldType.CHECKBOX && param == 1) { + fieldState.setChecked(true); + } + } + } } private boolean inList() { @@ -1384,16 +1385,29 @@ final class TextExtractor { // unicode RIGHT DOUBLE QUOTATION MARK addOutputChar('\u201D'); } - } else if (equals("fldinst")) { - fieldState = 1; - groupState.ignore = false; - } else if (equals("fldrslt") && fieldState == 2) { - assert pendingURL != null; - lazyStartParagraph(); - out.startElement("a", "href", pendingURL); - pendingURL = null; - fieldState = 3; + } else if (equals("field")) { + fieldState = new FieldState(); groupState.ignore = false; + } else if (fieldState != null) { + if (equals("fldinst")) { + fieldState.setCurrentGroup(FieldState.FieldGroups.fldinst); + groupState.ignore = false; + } else if (equals("formfield")) { + fieldState.setCurrentGroup(FieldState.FieldGroups.formfield); + groupState.ignore = false; + } else if (equals("ffname")) { + fieldState.setCurrentGroup(FieldState.FieldGroups.ffname); + groupState.ignore = false; + } else if (equals("fldrslt")) { + fieldState.setCurrentGroup(FieldState.FieldGroups.fldrslt); + + // If we don't know this type of field we'll simply output the text + if (fieldState.getInst() == FieldState.FieldInst.UNKNOWN) { + fieldState.setBufferText(false); + } + + groupState.ignore = false; + } } } @@ -1507,34 +1521,125 @@ final class TextExtractor { } assert groupStates.size() == groupState.depth; - if (fieldState == 1) { - String s = pendingBuffer.toString().trim(); - pendingBuffer.setLength(0); - if (s.startsWith("HYPERLINK")) { - s = s.substring(9).trim(); - // TODO: what other instructions can be in a - // HYPERLINK destination? - final boolean isLocalLink = s.contains("\\l "); - int idx = s.indexOf('"'); - if (idx != -1) { - int idx2 = s.indexOf('"', 1 + idx); - if (idx2 != -1) { - s = s.substring(1 + idx, idx2); - } - } - pendingURL = (isLocalLink ? "#" : "") + s; - fieldState = 2; + // Handle field destinations + if (fieldState != null) { + if (fieldState.getCurrentGroup() == null) { + // closing the field group + fieldState = null; } else { - fieldState = 0; + switch (fieldState.getCurrentGroup()) { + case fldinst: + processFieldInst(); + break; + + case fldrslt: + processFieldRslt(); + fieldState.setCurrentGroup(null); + break; + + case ffname: + if (pendingBuffer.length() > 0) { + String s = pendingBuffer.toString().trim(); + pendingBuffer.setLength(0); + fieldState.setName(s); + } + break; + + default: + fieldState.setBufferText(false); + } } + } + } + + private void processFieldInst() { + if (pendingBuffer.length() == 0) { + return; + } + + String s = pendingBuffer.toString().trim(); + pendingBuffer.setLength(0); + if (s.startsWith("HYPERLINK")) { + fieldState.setInst(FieldState.FieldInst.HYPERLINK); + s = s.substring(9).trim(); + // TODO: what other instructions can be in a + // HYPERLINK destination? + final boolean isLocalLink = s.contains("\\l "); + int idx = s.indexOf('"'); + if (idx != -1) { + int idx2 = s.indexOf('"', 1 + idx); + if (idx2 != -1) { + s = s.substring(1 + idx, idx2); + } + } + + fieldState.setUrl((isLocalLink ? "#" : "") + s); + } else if (s.startsWith("FORMCHECKBOX")) { + fieldState.setInst(FieldState.FieldInst.FORMCHECKBOX); + } else if (s.startsWith("FORMTEXT")) { + fieldState.setInst(FieldState.FieldInst.FORMTEXT); + } else { + fieldState.setInst(FieldState.FieldInst.UNKNOWN); + } + } + + /* This is where the output for a field actually happens. \fldrslt is the last group in the field destination */ + private void processFieldRslt() throws TikaException, SAXException, IOException { + switch (fieldState.getInst()) { + case HYPERLINK: + // Create Link text + String s = pendingBuffer.toString().trim(); + pendingBuffer.setLength(0); + fieldState.setResult(s); + + assert fieldState.getUrl() != null; + lazyStartParagraph(); + out.startElement("a", "href", fieldState.getUrl()); + out.characters(fieldState.getResult()); + end("a"); + break; + + case FORMCHECKBOX: + // Create + AttributesImpl checkboxAttributes = new AttributesImpl(); + checkboxAttributes.addAttribute("", "type", "type", "CDATA", "checkbox"); + checkboxAttributes.addAttribute("", "name", "name", "CDATA", fieldState.getName()); - // TODO: we could process the other known field - // types. Right now, we will extract their text - // inlined, but fail to record them in metadata - // as a field value. - } else if (fieldState == 3) { - end("a"); - fieldState = 0; + if (fieldState.isChecked()) { + checkboxAttributes.addAttribute("", "checked", "checked", "CDATA", "true"); + } + + out.startElement("input", checkboxAttributes); + out.endElement("input"); + break; + + case FORMTEXT: + // Create + s = pendingBuffer.toString().trim(); + pendingBuffer.setLength(0); + fieldState.setResult(s); + + AttributesImpl textAttributes = new AttributesImpl(); + textAttributes.addAttribute("", "type", "type", "CDATA", "text"); + textAttributes.addAttribute("", "name", "name", "CDATA", fieldState.getName()); + + if (fieldState.getResult() == null) { + textAttributes.addAttribute("", "value", "value", "CDATA", ""); + } else { + textAttributes.addAttribute("", "value", "value", "CDATA", fieldState.getResult()); + } + + out.startElement("input", textAttributes); + out.endElement("input"); + break; + + case UNKNOWN: + // we should never get here, but just in case we'll output anything in the pending buffer + if (pendingBuffer.length() > 0) { + out.characters(pendingBuffer.toString()); + } + + break; } } } diff --git tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java index 9c48d5602..a05e19ec8 100644 --- tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java +++ tika-parsers/src/test/java/org/apache/tika/parser/rtf/RTFParserTest.java @@ -16,22 +16,6 @@ */ package org.apache.tika.parser.rtf; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import org.apache.commons.io.FilenameUtils; import org.apache.tika.Tika; import org.apache.tika.TikaTest; @@ -39,11 +23,7 @@ import org.apache.tika.config.TikaConfig; import org.apache.tika.extractor.ContainerExtractor; import org.apache.tika.extractor.ParserContainerExtractor; import org.apache.tika.io.TikaInputStream; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.metadata.Office; -import org.apache.tika.metadata.OfficeOpenXMLCore; -import org.apache.tika.metadata.RTFMetadata; -import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.*; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; @@ -55,6 +35,14 @@ import org.apache.tika.sax.RecursiveParserWrapperHandler; import org.apache.tika.sax.WriteOutContentHandler; import org.junit.Test; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.*; + +import static org.junit.Assert.*; + /** * Junit test class for the Tika {@link RTFParser} */ @@ -590,6 +578,25 @@ public class RTFParserTest extends TikaTest { getXML("testRTFTIKA_2899.rtf").xml); } + @Test + public void testFormFields() throws Exception { + // Contains checkbox and text fields with text + XMLResult result = getXML("testRTFFormFields.rtf"); + + assertContains("", result.xml); + assertContains("", result.xml); + assertContains("", result.xml); + assertContains("", result.xml); + + + // Contains checkboxes and text fields and hyperlinks + XMLResult result2 = getXML("testRTFFormFields2.rtf"); + + assertContains("", result2.xml); + assertContains("", result2.xml); + assertContains("Add_Diagnosis_Description", result2.xml); + } + private Result getResult(String filename) throws Exception { File file = getResourceAsFile("/test-documents/" + filename); diff --git tika-parsers/src/test/resources/test-documents/testRTFFormFields.rtf tika-parsers/src/test/resources/test-documents/testRTFFormFields.rtf index bd7f4aae1..b49966fbb 100644 --- tika-parsers/src/test/resources/test-documents/testRTFFormFields.rtf +++ tika-parsers/src/test/resources/test-documents/testRTFFormFields.rtf @@ -237,9 +237,9 @@ \paperw12240\paperh15840\margl360\margr360\margt288\margb288\headery720\footery720\nogrowautofit\deftab720\formshade\fet4\aendnotes\aftnnrlc\pgbrdrhead\pgbrdrfoot \sectd\pgwsxn12240\pghsxn15840\marglsxn360\margrsxn360\margtsxn288\margbsxn288\headery720\footery720\sbkpage\pgncont\pgndec \plain\plain\f0\fs16\pard\ssparaaux0\s0\ltrpar\qc\widctlpar\plain\f0\fs16\plain\f1\fs24\lang1033\hich\f1\dbch\f1\loch\f1\fs24\b SMALL ANIMAL GENERAL WORK UP\par -\fs18\b0 Date: 5/28/2013\fs20\tab\tab\tab\tab\tab\tab\tab\tab Daniel and Anadele Roulain\tab\tab 9613\tab\par -\fs18 Patient Name: \fs28 Domanick \fs18 \tab 11 Yrs. 7 Mos.\tab DOBERMAN\tab 87.6 pounds\tab Neutered Male \par -\fs28 Dr. PATTERSON IV, DVM Technician: CP\fs18 \fs28\b CSR KT\cf1\fs18\b0\par +\fs18\b0 Date: 5/28/2013\fs20\tab\tab\tab\tab\tab\tab\tab\tab \par +\fs18 Patient Name: \fs28 \fs18 \tab 11 Yrs. 7 Mos.\tab DOBERMAN\tab 87.6 pounds\tab Neutered Male \par +\fs28 Dr. DVM Technician: CP\fs18 \fs28\b CSR KT\cf1\fs18\b0\par \trowd\trgaph60\trleft0\trrh574 \clvertalt\clbrdrt\brdrs\brdrw20\clbrdrb\brdrs\brdrw20\clbrdrl\brdrs\brdrw20\clbrdrr\brdrs\brdrw20\clcbpat2\cellx11430 \pard\intbl\ssparaaux0\s0\ltrpar\ql\widctlpar\plain\f0\fs16\plain\f1\fs20\lang1033\hich\f1\dbch\f1\loch\f1\cf1\fs20\b\ul SUBJECTIVE CC\ul0 : 8-9 days ago noticed injury to 4th digit on L front foot, owner was applying neo predef powder and it seemed to @@ -465,4 +465,4 @@ Prescriptions: \hich\f2\dbch\f2\loch\f2\fs16\sscharaux70013\b0\i{\field{\*\fldin Plan: \hich\f2\dbch\f2\loch\f2\fs16\sscharaux70014\b0\i{\field{\*\fldinst{FORMTEXT}{\*\formfield\fftype0{\*\ffname PLAN}}}{\fldrslt _Need to recheck toe and ears in 2 weeks. scheduled for 6/12/13 at 10 am. Owner to cont using neo predef powder on toe and is okay to refill it if she runs out_}}\hich\f1\dbch\f1\loch\f1\fs18\sscharaux0\b\i0\pard\par } - \ No newline at end of file + diff --git tika-parsers/src/test/resources/test-documents/testRTFFormFields2.rtf tika-parsers/src/test/resources/test-documents/testRTFFormFields2.rtf index c317f7f5f..1e67a7fbe 100644 --- tika-parsers/src/test/resources/test-documents/testRTFFormFields2.rtf +++ tika-parsers/src/test/resources/test-documents/testRTFFormFields2.rtf @@ -237,9 +237,9 @@ \paperw12240\paperh15840\margl360\margr360\margt288\margb288\headery720\footery720\nogrowautofit\deftab720\formshade\fet4\aendnotes\aftnnrlc\pgbrdrhead\pgbrdrfoot \sectd\pgwsxn12240\pghsxn15840\marglsxn360\margrsxn360\margtsxn288\margbsxn288\headery720\footery720\sbkpage\pgncont\pgndec \plain\plain\f0\fs20\pard\ssparaaux0\s0\ltrpar\qc\widctlpar\plain\f0\fs20\plain\f1\fs24\lang1033\hich\f1\dbch\f1\loch\f1\fs24\b SMALL ANIMAL GENERAL WORK UP\par -\fs18\b0 Date: 5/19/2015\fs20\tab\tab\tab\tab\tab\tab\tab\tab Les and Denise Ferguson\tab\tab 9612\tab\par -\fs18 Patient Name: \fs28 Seguin \fs18 \tab 10 Yrs. 0 Mos.\tab DOMESTIC SHORTHAIR\tab 10 pounds\tab Neutered Male \par -\fs28 Dr. PATTERSON IV, DVM Technician: OS\fs18 \fs28\b CSR \cf1\fs18\b0\par +\fs18\b0 Date: 5/19/2015\fs20\tab\tab\tab\tab\tab\tab\tab\tab \par +\fs18 Patient Name: \tab 10 Yrs. 0 Mos.\tab DOMESTIC SHORTHAIR\tab 10 pounds\tab Neutered Male \par +\fs28 Dr. DVM Technician: OS\fs18 \fs28\b CSR \cf1\fs18\b0\par \trowd\trgaph60\trleft0\trrh574 \clvertalt\clbrdrt\brdrs\brdrw20\clbrdrb\brdrs\brdrw20\clbrdrl\brdrs\brdrw20\clbrdrr\brdrs\brdrw20\clcbpat2\cellx11430 \pard\intbl\ssparaaux0\s0\ltrpar\ql\widctlpar\plain\f0\fs20\plain\f1\fs20\lang1033\hich\f1\dbch\f1\loch\f1\cf1\fs20\b\ul SUBJECTIVE CC\ul0 : WELLNESS/VACCS/GROWTH BEHIND L EAR\par @@ -389,4 +389,4 @@ Prescriptions: \hich\f2\dbch\f2\loch\f2\fs16\sscharaux70016\b0\i{\field{\*\fldin Plan: \hich\f2\dbch\f2\loch\f2\fs16\sscharaux70017\b0\i{\field{\*\fldinst{FORMTEXT}{\*\formfield\fftype0{\*\ffname PLAN}}}{\fldrslt __}}\hich\f1\dbch\f1\loch\f1\fs18\sscharaux0\b\i0 MONITOR GROWTH FOR CHANGES. CONTINUE WITH ANNUAL EXAMS.\pard\par \par } -�rd \ No newline at end of file +�rd