+ * This constructor is available to allow for Tika's legacy ( <= v1.7) behavior. + */ + public AccessChecker() { + needToCheck = false; + allowAccessibility = true; + } + /** + * This constructs an {@link AccessChecker} that will check + * for whether or not content should be extracted from a document. + * + * @param allowExtractionForAccessibility if general extraction is not allowed, is extraction for accessibility allowed + */ + public AccessChecker(boolean allowExtractionForAccessibility) { + needToCheck = true; + this.allowAccessibility = allowExtractionForAccessibility; + } + + /** + * Checks to see if a document's content should be extracted based + * on metadata values and the value of {@link #allowAccessibility} in the constructor. + * + * @param metadata + * @throws AccessPermissionException if access is not permitted + */ + public void check(Metadata metadata) throws AccessPermissionException { + if (!needToCheck) { + return; + } + if ("false".equals(metadata.get(AccessPermissions.EXTRACT_CONTENT))) { + if (allowAccessibility) { + if("true".equals(metadata.get(AccessPermissions.EXTRACT_FOR_ACCESSIBILITY))) { + return; + } + throw new AccessPermissionException("Content extraction for accessibility is not allowed."); + } + throw new AccessPermissionException("Content extraction is not allowed."); + } + } +} Index: tika-core/src/main/java/org/apache/tika/exception/AccessPermissionException.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/exception/AccessPermissionException.java (revision ) +++ tika-core/src/main/java/org/apache/tika/exception/AccessPermissionException.java (revision ) @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.exception; + +/** + * Exception to be thrown when a document does not allow content extraction. + * As of this writing, PDF documents are the only type of document that might + * cause this type of exception. + */ +public class AccessPermissionException extends TikaException { + public AccessPermissionException() { + super("Unable to process: content extraction is not allowed"); + } + + public AccessPermissionException(Throwable th) { + super("Unable to process: content extraction is not allowed", th); + } + + public AccessPermissionException(String info) { + super(info); + } + + public AccessPermissionException(String info, Throwable th) { + super(info, th); + } +} Index: tika-core/src/main/java/org/apache/tika/metadata/AccessPermissions.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/AccessPermissions.java (revision ) +++ tika-core/src/main/java/org/apache/tika/metadata/AccessPermissions.java (revision ) @@ -0,0 +1,71 @@ +package org.apache.tika.metadata; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Until we can find a common standard, we'll use these options. They + * were mostly derived from PDFBox's AccessPermission, but some can + * apply to other document formats, especially CAN_MODIFY and FILL_IN_FORM. + */ +public interface AccessPermissions { + + final static String PREFIX = "access_permission"+Metadata.NAMESPACE_PREFIX_DELIMITER; + + /** + * Can any modifications be made to the document + */ + Property CAN_MODIFY = Property.externalTextBag(PREFIX+"can_modify"); + + /** + * Should content be extracted, generally. + */ + Property EXTRACT_CONTENT = Property.externalText(PREFIX+"extract_content"); + + /** + * Should content be extracted for the purposes + * of accessibility. + */ + Property EXTRACT_FOR_ACCESSIBILITY = Property.externalText(PREFIX + "extract_for_accessibility"); + + /** + * Can the user insert/rotate/delete pages. + */ + Property ASSEMBLE_DOCUMENT = Property.externalText(PREFIX+"assemble_document"); + + + /** + * Can the user fill in a form + */ + Property FILL_IN_FORM = Property.externalText(PREFIX+"fill_in_form"); + + /** + * Can the user modify annotations + */ + Property CAN_MODIFY_ANNOTATIONS = Property.externalText(PREFIX+"modify_annotations"); + + /** + * Can the user print the document + */ + Property CAN_PRINT = Property.externalText(PREFIX+"can_print"); + + /** + * Can the user print an image-degraded version of the document. + */ + Property CAN_PRINT_DEGRADED = Property.externalText(PREFIX+"can_print_degraded"); + +} Index: tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java (date 1424460582000) +++ tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java (revision ) @@ -17,14 +17,14 @@ * limitations under the License. */ +import org.apache.pdfbox.util.PDFTextStripper; + import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.util.Locale; import java.util.Properties; -import org.apache.pdfbox.util.PDFTextStripper; - /** * Config for PDFParser. * @@ -80,6 +80,8 @@ //The space width-based tolerance value used to estimate where spaces in text should be added private Float spacingTolerance; + private AccessChecker accessChecker; + public PDFParserConfig() { init(this.getClass().getResourceAsStream("PDFParser.properties")); } @@ -137,7 +139,18 @@ setExtractUniqueInlineImagesOnly( getProp(props.getProperty("extractUniqueInlineImagesOnly"), getExtractUniqueInlineImagesOnly())); + + boolean checkExtractAccessPermission = getProp(props.getProperty("checkExtractAccessPermission"), false); + boolean allowExtractionForAccessibility = getProp(props.getProperty("allowExtractionForAccessibility"), true); + + if (checkExtractAccessPermission == false) { + //silently ignore the crazy configuration of checkExtractAccessPermission = false, + //but allowExtractionForAccessibility=false + accessChecker = new AccessChecker(); + } else { + accessChecker = new AccessChecker(allowExtractionForAccessibility); - } + } + } /** * Configures the given pdf2XHTML. @@ -332,6 +345,14 @@ */ public void setSpacingTolerance(Float spacingTolerance) { this.spacingTolerance = spacingTolerance; + } + + public void setAccessChecker(AccessChecker accessChecker) { + this.accessChecker = accessChecker; + } + + public AccessChecker getAccessChecker() { + return accessChecker; } private boolean getProp(String p, boolean defaultMissing){ Index: tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java (date 1424460582000) +++ tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java (revision ) @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.tika.TikaTest; +import org.apache.tika.exception.AccessPermissionException; import org.apache.tika.exception.EncryptedDocumentException; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.ContainerExtractor; @@ -642,17 +643,20 @@ continue; } - pdfs++; - String sequentialContent = null; Metadata sequentialMetadata = new Metadata(); try { sequentialContent = getText(new FileInputStream(f), sequentialParser, seqContext, sequentialMetadata); + } catch (EncryptedDocumentException e) { + //silently skip a file that requires a user password + continue; } catch (Exception e) { throw new TikaException("Sequential Parser failed on test file " + f, e); } + pdfs++; + String nonSequentialContent = null; Metadata nonSequentialMetadata = new Metadata(); try { @@ -1138,6 +1142,202 @@ assertContains("
", xml); } + //Access checker tests + + @Test + public void testLegacyAccessChecking() throws Exception { + //test that default behavior doesn't throw AccessPermissionException + for (String file : new String[] { + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + "testPDF_no_extract_yes_accessibility_owner_empty.pdf", + }) { + String xml = getXML(file).xml; + assertContains("Hello World", xml); + } + + //now try with the user password + PasswordProvider provider = new PasswordProvider() { + @Override + public String getPassword(Metadata metadata) { + return "user"; + } + }; + + ParseContext context = new ParseContext(); + context.set(PasswordProvider.class, provider); + Parser parser = new AutoDetectParser(); + + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_user.pdf", + "testPDF_no_extract_yes_accessibility_owner_user.pdf", + }) { + InputStream stream = null; + try { + stream = TikaInputStream.get(this.getClass().getResource("/test-documents/"+path)); + String text = getText(stream, parser, context); + assertContains("Hello World", text); + } finally { + IOUtils.closeQuietly(stream); + } + } + } + + @Test + public void testAccessCheckingEmptyPassword() throws Exception { + PDFParserConfig config = new PDFParserConfig(); + + //don't allow extraction, not even for accessibility + config.setAccessChecker(new AccessChecker(false)); + Parser parser = new AutoDetectParser(); + ParseContext context = new ParseContext(); + context.set(PDFParserConfig.class, config); + + //test exception for empty password + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + "testPDF_no_extract_yes_accessibility_owner_empty.pdf", + }) { + assertException("/test-documents/"+path, parser, context, AccessPermissionException.class); + } + + config.setAccessChecker(new AccessChecker(true)); + assertException("/test-documents/" + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + parser, context, AccessPermissionException.class); + + InputStream is = null; + try { + is = getResourceAsStream("/test-documents/"+ "testPDF_no_extract_yes_accessibility_owner_empty.pdf"); + assertContains("Hello World", getText(is, parser, context)); + } finally { + IOUtils.closeQuietly(is); + } + } + + @Test + public void testAccessCheckingUserPassword() throws Exception { + ParseContext context = new ParseContext(); + + PDFParserConfig config = new PDFParserConfig(); + //don't allow extraction, not even for accessibility + config.setAccessChecker(new AccessChecker(false)); + PasswordProvider passwordProvider = new PasswordProvider() { + @Override + public String getPassword(Metadata metadata) { + return "user"; + } + }; + + context.set(PasswordProvider.class, passwordProvider); + context.set(PDFParserConfig.class, config); + + Parser parser = new AutoDetectParser(); + + //test bad passwords + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + "testPDF_no_extract_yes_accessibility_owner_empty.pdf", + }) { + assertException("/test-documents/"+path, parser, context, EncryptedDocumentException.class); + } + + //bad password is still a bad password + config.setAccessChecker(new AccessChecker(true)); + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + "testPDF_no_extract_yes_accessibility_owner_empty.pdf", + }) { + assertException("/test-documents/"+path, parser, context, EncryptedDocumentException.class); + } + + //now test documents that require this "user" password + assertException("/test-documents/"+"testPDF_no_extract_no_accessibility_owner_user.pdf", + parser, context, AccessPermissionException.class); + + + InputStream is = null; + try { + is = getResourceAsStream("/test-documents/"+ "testPDF_no_extract_yes_accessibility_owner_user.pdf"); + assertContains("Hello World", getText(is, parser, context)); + } finally { + IOUtils.closeQuietly(is); + } + + config.setAccessChecker(new AccessChecker(false)); + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_user.pdf", + "testPDF_no_extract_yes_accessibility_owner_user.pdf", + }) { + assertException("/test-documents/"+path, parser, context, AccessPermissionException.class); + } + } + + @Test + public void testAccessCheckingOwnerPassword() throws Exception { + ParseContext context = new ParseContext(); + + PDFParserConfig config = new PDFParserConfig(); + //don't allow extraction, not even for accessibility + config.setAccessChecker(new AccessChecker(true)); + PasswordProvider passwordProvider = new PasswordProvider() { + @Override + public String getPassword(Metadata metadata) { + return "owner"; + } + }; + + context.set(PasswordProvider.class, passwordProvider); + context.set(PDFParserConfig.class, config); + + Parser parser = new AutoDetectParser(); + //with owner's password, text can be extracted, no matter the AccessibilityChecker's settings + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_user.pdf", + "testPDF_no_extract_yes_accessibility_owner_user.pdf", + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + "testPDF_no_extract_yes_accessibility_owner_empty.pdf", + }) { + + InputStream is = null; + try { + is = getResourceAsStream("/test-documents/" + "testPDF_no_extract_yes_accessibility_owner_user.pdf"); + assertContains("Hello World", getText(is, parser, context)); + } finally { + IOUtils.closeQuietly(is); + } + } + + //really, with owner's password, all extraction is allowed + config.setAccessChecker(new AccessChecker(false)); + for (String path : new String[] { + "testPDF_no_extract_no_accessibility_owner_user.pdf", + "testPDF_no_extract_yes_accessibility_owner_user.pdf", + "testPDF_no_extract_no_accessibility_owner_empty.pdf", + "testPDF_no_extract_yes_accessibility_owner_empty.pdf", + }) { + + InputStream is = null; + try { + is = getResourceAsStream("/test-documents/" + "testPDF_no_extract_yes_accessibility_owner_user.pdf"); + assertContains("Hello World", getText(is, parser, context)); + } finally { + IOUtils.closeQuietly(is); + } + } + } + + private void assertException(String path, Parser parser, ParseContext context, Class expected) { + boolean noEx = false; + InputStream is = getResourceAsStream(path); + try { + String text = getText(is, parser, context); + noEx = true; + } catch (Exception e) { + assertEquals("Not the right exception: "+path, expected, e.getClass()); + } finally { + IOUtils.closeQuietly(is); + } + assertFalse(path + " should have thrown exception", noEx); + } /** * * Simple class to count end of document events. If functionality is useful, Index: tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>windows-1252 =================================================================== --- tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties (date 1424460582000) +++ tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties (revision ) @@ -21,3 +21,5 @@ extractAcroFormContent true extractInlineImages false extractUniqueInlineImagesOnly true +checkExtractAccessPermission false +allowExtractionForAccessibility true Index: tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java (date 1424460582000) +++ tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java (revision ) @@ -39,6 +39,7 @@ import org.apache.pdfbox.io.RandomAccessFile; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocumentInformation; +import org.apache.pdfbox.pdmodel.encryption.AccessPermission; import org.apache.pdfbox.pdmodel.font.PDFont; import org.apache.tika.exception.EncryptedDocumentException; import org.apache.tika.exception.TikaException; @@ -46,6 +47,7 @@ import org.apache.tika.io.CloseShieldInputStream; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.AccessPermissions; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.PagedText; import org.apache.tika.metadata.Property; @@ -140,6 +142,9 @@ metadata.set(Metadata.CONTENT_TYPE, "application/pdf"); extractMetadata(pdfDocument, metadata); + + AccessChecker checker = localConfig.getAccessChecker(); + checker.check(metadata); if (handler != null) { PDF2XHTML.process(pdfDocument, handler, context, metadata, localConfig); } @@ -191,6 +196,28 @@ private void extractMetadata(PDDocument document, Metadata metadata) throws TikaException { + //first extract AccessPermissions + AccessPermission ap = document.getCurrentAccessPermission(); + metadata.set(AccessPermissions.EXTRACT_FOR_ACCESSIBILITY, + Boolean.toString(ap.canExtractForAccessibility())); + metadata.set(AccessPermissions.EXTRACT_CONTENT, + Boolean.toString(ap.canExtractContent())); + metadata.set(AccessPermissions.ASSEMBLE_DOCUMENT, + Boolean.toString(ap.canAssembleDocument())); + metadata.set(AccessPermissions.FILL_IN_FORM, + Boolean.toString(ap.canFillInForm())); + metadata.set(AccessPermissions.CAN_MODIFY, + Boolean.toString(ap.canModify())); + metadata.set(AccessPermissions.CAN_MODIFY_ANNOTATIONS, + Boolean.toString(ap.canModifyAnnotations())); + metadata.set(AccessPermissions.CAN_PRINT, + Boolean.toString(ap.canPrint())); + metadata.set(AccessPermissions.CAN_PRINT_DEGRADED, + Boolean.toString(ap.canPrintDegraded())); + + + + //now go for the XMP stuff org.apache.jempbox.xmp.XMPMetadata xmp = null; XMPSchemaDublinCore dcSchema = null; try{