Index: tika-core/src/main/java/org/apache/tika/mime/MimeType.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/MimeType.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/MimeType.java (working copy) @@ -19,7 +19,6 @@ import java.io.Serializable; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -29,7 +28,7 @@ public final class MimeType implements Comparable, Serializable { /** - * Serial version UID. + * Generated serial ID */ private static final long serialVersionUID = 4357830439860729201L; Index: tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java (working copy) @@ -27,6 +27,11 @@ */ class MagicMatch implements Clause { + /** + * Generated serial ID + */ + private static final long serialVersionUID = 4693039941295960620L; + private final MediaType mediaType; private final String type; Index: tika-core/src/main/java/org/apache/tika/mime/AndClause.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/AndClause.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/AndClause.java (working copy) @@ -20,6 +20,10 @@ class AndClause implements Clause { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -3572009507680757629L; private final Clause[] clauses; AndClause(Clause... clauses) { Index: tika-core/src/main/java/org/apache/tika/mime/purifier/package-info.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/purifier/package-info.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/mime/purifier/package-info.java (revision 0) @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains all the logic to implement your own + * {@link org.apache.tika.mime.purifier.Purifier}. + * Each {@link org.apache.tika.mime.purifier.Purifier} is + * responsible for modifying the file before + * its MIME type is detected. + */ +package org.apache.tika.mime.purifier; \ No newline at end of file Index: tika-core/src/main/java/org/apache/tika/mime/purifier/WhiteSpacesPurifier.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/purifier/WhiteSpacesPurifier.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/mime/purifier/WhiteSpacesPurifier.java (revision 0) @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tika.mime.purifier; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Implementation of {@link org.apache.tika.mime.Purifier} that removes + * all the eventual blank characters at the header of a file that + * might prevents its MIME Type detection. + * + * @author Davide Palmisano ( dpalmisano@gmail.com ) + */ +public class WhiteSpacesPurifier implements Purifier { + + /** + * {@inheritDoc} + */ + public void purify(InputStream inputStream) throws IOException { + if(!inputStream.markSupported()) + throw new IllegalArgumentException( + "Provided InputStream does not support marks"); + + // mark the current position + inputStream.mark(Integer.MAX_VALUE); + int byteRead = inputStream.read(); + char charRead = (char) byteRead; + while(isBlank(charRead) && (byteRead != -1)) { + // if here means that the previous character must be removed, so mark. + inputStream.mark(Integer.MAX_VALUE); + byteRead = inputStream.read(); + charRead = (char) byteRead; + } + // if exit go back to the last valid mark. + inputStream.reset(); + } + + private boolean isBlank(char c) { + return c == '\t' || c == '\n' || c == ' ' || c == '\r' || c == '\b' || c == '\f'; + } +} Index: tika-core/src/main/java/org/apache/tika/mime/purifier/Purifier.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/purifier/Purifier.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/mime/purifier/Purifier.java (revision 0) @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tika.mime.purifier; + +import java.io.IOException; +import java.io.InputStream; + +/** + * This interface defines a minimum set of methods that + * a {@link org.apache.tika.detect.Detector} could + * call in order to clean the input before performing the MIME type + * detection. + * + * @author Davide Palmisano ( dpalmisano@gmail.com ) + */ +public interface Purifier { + + /** + * Performs the purification of the provided resettable {@link java.io.InputStream}. + * + * @param inputStream a resettable {@link java.io.InputStream} to be cleaned. + */ + void purify(InputStream inputStream) throws IOException; + +} Index: tika-core/src/main/java/org/apache/tika/mime/MimeTypeException.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/MimeTypeException.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/MimeTypeException.java (working copy) @@ -23,6 +23,11 @@ */ public class MimeTypeException extends TikaException { + /** + * Default serial ID + */ + private static final long serialVersionUID = -1122922305859821242L; + /** * Constructs a MimeTypeException with the specified detail message. * Index: tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java (working copy) @@ -38,6 +38,8 @@ import org.apache.tika.detect.TextDetector; import org.apache.tika.detect.XmlRootExtractor; import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.purifier.Purifier; +import org.apache.tika.mime.purifier.WhiteSpacesPurifier; /** * This class is a MimeType repository. It gathers a set of MimeTypes and @@ -161,6 +163,9 @@ *

* The given byte array is expected to be at least {@link #getMinLength()} * long, or shorter only if the document stream itself is shorter. + *

+ * If a purifier is present then this is used prior to {@link #getMinLength()} + * processing. * * @param data first few bytes of a document stream * @return matching MIME type @@ -212,6 +217,14 @@ try { TextDetector detector = new TextDetector(getMinLength()); ByteArrayInputStream stream = new ByteArrayInputStream(data); + if(stream != null) { + try { + Purifier purifier = new WhiteSpacesPurifier(); + purifier.purify(stream); + } catch (IOException e) { + throw new RuntimeException("Error while purifying the provided input", e); + } + } return forName(detector.detect(stream, new Metadata()).toString()); } catch (Exception e) { return rootMimeType; @@ -531,4 +544,5 @@ } return types; } + } Index: tika-core/src/main/java/org/apache/tika/mime/OrClause.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/OrClause.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/OrClause.java (working copy) @@ -20,7 +20,11 @@ class OrClause implements Clause { - private final List clauses; + /** + * Generated serial ID + */ + private static final long serialVersionUID = 2089222390646997974L; + private final List clauses; OrClause(List clauses) { this.clauses = clauses; Index: tika-core/src/main/java/org/apache/tika/mime/Magic.java =================================================================== --- tika-core/src/main/java/org/apache/tika/mime/Magic.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/mime/Magic.java (working copy) @@ -24,6 +24,11 @@ */ class Magic implements Clause, Comparable { + /** + * Generated serial ID + */ + private static final long serialVersionUID = 5308393371823772332L; + private final MimeType type; private final int priority; Index: tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java =================================================================== --- tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java (working copy) @@ -26,6 +26,10 @@ public class TaggedSAXException extends SAXException { /** + * Generated serial ID + */ + private static final long serialVersionUID = -2383105081515353584L; + /** * The object reference used to tag the exception. */ private final Object tag; Index: tika-core/src/main/java/org/apache/tika/parser/external/ExternalParsersFactory.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/external/ExternalParsersFactory.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/parser/external/ExternalParsersFactory.java (working copy) @@ -82,7 +82,8 @@ Parser parser = config.getParser(); if (parser instanceof CompositeParser) { CompositeParser cParser = (CompositeParser)parser; - Map parserMap = cParser.getParsers(); + @SuppressWarnings("unused") + Map parserMap = cParser.getParsers(); } // TODO } Index: tika-core/src/main/java/org/apache/tika/parser/ParserPostProcessor.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/ParserPostProcessor.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/parser/ParserPostProcessor.java (working copy) @@ -36,6 +36,11 @@ public class ParserPostProcessor extends ParserDecorator { /** + * Generated serial ID + */ + private static final long serialVersionUID = 1082816055239659736L; + + /** * Creates a post-processing decorator for the given parser. * * @param parser the parser to be decorated Index: tika-core/src/main/java/org/apache/tika/parser/ErrorParser.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/ErrorParser.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/parser/ErrorParser.java (working copy) @@ -33,6 +33,10 @@ public class ErrorParser extends AbstractParser { /** + * Generated serial ID + */ + private static final long serialVersionUID = 7727423956957641824L; + /** * Singleton instance of this class. */ public static final ErrorParser INSTANCE = new ErrorParser(); Index: tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java (working copy) @@ -43,6 +43,11 @@ public class NetworkParser extends AbstractParser { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -5416099113508995890L; + private final URI uri; private final Set supportedTypes; Index: tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java (working copy) @@ -28,6 +28,8 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MediaTypeRegistry; +import org.apache.tika.mime.purifier.Purifier; +import org.apache.tika.mime.purifier.WhiteSpacesPurifier; import org.apache.tika.sax.SecureContentHandler; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; @@ -51,29 +53,53 @@ this(TikaConfig.getDefaultConfig()); } + /** + * Creates an auto-detecting parser instance using the default Tika + * configuration with the addition of a specific + * {@link org.apache.tika.detect.Detector} implementation. + * @param detector a user defined detector. + */ public AutoDetectParser(Detector detector) { this(TikaConfig.getDefaultConfig()); setDetector(detector); } /** - * Creates an auto-detecting parser instance using the specified set of parser. + * Creates an auto-detecting parser instance using the specified set of parser(s) + * and the default {@link org.apache.tika.detect.DefaultDetector}. * This allows one to create a Tika configuration where only a subset of the * available parsers have their 3rd party jars included, as otherwise the * use of the default TikaConfig will throw various "ClassNotFound" exceptions. * - * @param detector Detector to use - * @param parsers + * @param parsers the parsers to include. */ public AutoDetectParser(Parser...parsers) { this(new DefaultDetector(), parsers); } + /** + * Creates an auto-detecting parser instance using the specified set + * and the default {@link org.apache.tika.detect.Detector}(s) and parser(s) + * implementations. + * This allows one to create a Tika configuration where only a subset of the + * available parsers have their 3rd party jars included, as otherwise the + * use of the default TikaConfig will throw various "ClassNotFound" exceptions. + * + * @param parsers the parsers to include. + */ public AutoDetectParser(Detector detector, Parser...parsers) { super(MediaTypeRegistry.getDefaultRegistry(), parsers); setDetector(detector); } + /** + * Create an auto-detecting parser using an existing + * {@link org.apache.tika.config.TikaConfig} instance. + * Subsequently we obtain {@link org.apache.tika.detect.Detector} and + * {@link org.apache.tika.parser.Parser} implementations + * from the configuration. + * @param config the existing Tika configuration. + */ public AutoDetectParser(TikaConfig config) { super(config.getMediaTypeRegistry(), config.getParser()); setDetector(config.getDetector()); @@ -106,6 +132,14 @@ Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { TemporaryResources tmp = new TemporaryResources(); + if(stream != null) { + try { + Purifier purifier = new WhiteSpacesPurifier(); + purifier.purify(stream); + } catch (IOException e) { + throw new RuntimeException("Error while purifying the provided input", e); + } + } try { TikaInputStream tis = TikaInputStream.get(stream, tmp); Index: tika-core/src/main/java/org/apache/tika/parser/DelegatingParser.java =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/DelegatingParser.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/parser/DelegatingParser.java (working copy) @@ -37,6 +37,11 @@ public class DelegatingParser extends AbstractParser { /** + * Generated serial ID + */ + private static final long serialVersionUID = -3155013638985767859L; + + /** * Returns the parser instance to which parsing tasks should be delegated. * The default implementation looks up the delegate parser from the given * parse context, and uses an {@link EmptyParser} instance as a fallback. Index: tika-core/src/main/java/org/apache/tika/exception/EncryptedDocumentException.java =================================================================== --- tika-core/src/main/java/org/apache/tika/exception/EncryptedDocumentException.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/exception/EncryptedDocumentException.java (working copy) @@ -17,6 +17,11 @@ package org.apache.tika.exception; public class EncryptedDocumentException extends TikaException { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -4640940743548761225L; + public EncryptedDocumentException() { super("Unable to process: document is encrypted"); } Index: tika-core/src/main/java/org/apache/tika/exception/TikaException.java =================================================================== --- tika-core/src/main/java/org/apache/tika/exception/TikaException.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/exception/TikaException.java (working copy) @@ -21,6 +21,11 @@ */ public class TikaException extends Exception { + /** + * Generated serial ID + */ + private static final long serialVersionUID = 6169468760011493332L; + public TikaException(String msg) { super(msg); } Index: tika-core/src/main/java/org/apache/tika/fork/ForkObjectInputStream.java =================================================================== --- tika-core/src/main/java/org/apache/tika/fork/ForkObjectInputStream.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/fork/ForkObjectInputStream.java (working copy) @@ -105,6 +105,7 @@ byte[] data = new byte[n]; input.readFully(data); + @SuppressWarnings("resource") ObjectInputStream deserializer = new ForkObjectInputStream(new ByteArrayInputStream(data), loader); return deserializer.readObject(); Index: tika-core/src/main/java/org/apache/tika/io/TaggedIOException.java =================================================================== --- tika-core/src/main/java/org/apache/tika/io/TaggedIOException.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/io/TaggedIOException.java (working copy) @@ -26,6 +26,10 @@ public class TaggedIOException extends IOExceptionWithCause { /** + * Generated serial ID + */ + private static final long serialVersionUID = 7828785471783649782L; + /** * The object reference used to tag the exception. */ private final Object tag; Index: tika-core/src/main/java/org/apache/tika/detect/TextDetector.java =================================================================== --- tika-core/src/main/java/org/apache/tika/detect/TextDetector.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/detect/TextDetector.java (working copy) @@ -22,6 +22,8 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.purifier.Purifier; +import org.apache.tika.mime.purifier.WhiteSpacesPurifier; /** * Content type detection of plain text documents. This detector looks at the @@ -101,7 +103,9 @@ /** * Looks at the beginning of the document input stream to determine - * whether the document is text or not. + * whether the document is text or not. Uses the default + * {@link org.apache.tika.mime.purifier.WhiteSpacesPurifier()} + * on the InputStream. * * @param input document input stream, or null * @param metadata ignored @@ -113,7 +117,17 @@ if (input == null) { return MediaType.OCTET_STREAM; } - + + //Purify the InputStream with the 'default' + //WhiteSpacePurifier. In the future we could make + //this configurable. + Purifier purifier = new WhiteSpacesPurifier(); + try { + purifier.purify(input); + } catch (IOException e) { + throw new RuntimeException("Error while purifying the provided InputStream", e); + } + input.mark(bytesToTest); try { TextStatistics stats = new TextStatistics(); Index: tika-core/src/main/java/org/apache/tika/detect/Any23Detector.java =================================================================== --- tika-core/src/main/java/org/apache/tika/detect/Any23Detector.java (revision 0) +++ tika-core/src/main/java/org/apache/tika/detect/Any23Detector.java (revision 0) @@ -0,0 +1,246 @@ +/** + * + */ +package org.apache.tika.detect; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.regex.Pattern; + +import org.apache.tika.Tika; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.MimeTypes; +import org.apache.tika.mime.purifier.Purifier; +import org.openrdf.rio.RDFFormat; +import org.openrdf.rio.RDFParser; +import org.openrdf.rio.Rio; + +/** + * Implementation of {@link org.apache.tika.detect.Detector} + * based on Semantic web formats. + * + * @author Michele Mostarda (michele.mostarda@gmail.com) + * @author Davide Palmisano (dpalmisano@gmail.com) + */ +public class Any23Detector implements Detector { + + /** Generated serialID */ + private static final long serialVersionUID = 8673328556174564097L; + + private Purifier purifier; + + private Tika tika; + + public static final String CSV_MIMETYPE = "text/csv"; + + /** + * Default Constructor which uses the + * {@link org.apache.tika.mime.purifier.WhiteSpacesPurifier()} + */ + public Any23Detector() { + this( new org.apache.tika.mime.purifier.WhiteSpacesPurifier() ); + } + + /** + * Supplementary constructor offering the option for a custom + * {@link org.apache.tika.mime.purifier.Purifier()} + * @param purifier + */ + public Any23Detector(Purifier purifier) { + this.purifier = purifier; + } + + /** + * Estimates the MediaType of the input content. + * The input stream must be resettable. + * + * @param input null or a resettable input stream + * containing data. + * @param metadata mimetype declared in metadata. + * @param the purifier implementation. + * @return the supposed MediaType or null + * if nothing appropriate found. + * @throws IllegalArgumentException if input is not + * null and is not resettable. + */ + @Override + public MediaType detect(InputStream input, Metadata metadata) throws IOException { + + if(input != null) { + try { + this.purifier.purify(input); + } catch (IOException e) { + throw new RuntimeException("Error while purifying the provided input", e); + } + } + + String type; + tika = new Tika(); + try { + final String mt = tika.detect(input, metadata); + if( ! MimeTypes.OCTET_STREAM.equals(mt) ) { + type = mt; + } else { + if( checkN3Format(input) ) { + type = RDFFormat.N3.getDefaultMIMEType(); + } else if( checkNQuadsFormat(input) ) { + type = RDFFormat.NQUADS.getDefaultMIMEType(); + } else if( checkTurtleFormat(input) ) { + type = RDFFormat.TURTLE.getDefaultMIMEType(); + //TODO add once CSV module has been ported from Any23 + //} else if( checkCSVFormat(input) ) { + // type = CSV_MIMETYPE; + } + else { + type = MimeTypes.OCTET_STREAM; + } + } + } catch (IOException ioe) { + throw new RuntimeException("Error while retrieving mime type.", ioe); + } + return MediaType.parse(type); + } + + /** + * N3 patterns. + */ + private static final Pattern[] N3_PATTERNS = { + Pattern.compile("^\\S+\\s*<\\S+>\\s*<\\S+>\\s*\\." ), // * URI URI . + Pattern.compile("^\\S+\\s*<\\S+>\\s*_:\\S+\\s*\\." ), // * URI BNODE . + Pattern.compile("^\\S+\\s*<\\S+>\\s*\".*\"(@\\S+)?\\s*\\." ), // * URI LLITERAL . + Pattern.compile("^\\S+\\s*<\\S+>\\s*\".*\"(\\^\\^\\S+)?\\s*\\.") // * URI TLITERAL . + }; + + /** + * N-Quads patterns. + */ + private static final Pattern[] NQUADS_PATTERNS = { + Pattern.compile("^\\S+\\s*<\\S+>\\s*<\\S+>\\s*\\<\\S+>\\s*\\." ), // * URI URI URI . + Pattern.compile("^\\S+\\s*<\\S+>\\s*_:\\S+\\s*\\<\\S+>\\s*\\." ), // * URI BNODE URI . + Pattern.compile("^\\S+\\s*<\\S+>\\s*\".*\"(@\\S+)?\\s*\\<\\S+>\\s*\\." ), // * URI LLITERAL URI . + Pattern.compile("^\\S+\\s*<\\S+>\\s*\".*\"(\\^\\^\\S+)?\\s*\\<\\S+>\\s*\\.") // * URI TLITERAL URI . + }; + + /** + * Checks if the stream contains the N3 triple patterns. + * + * @param is input stream to be verified. + * @return true if N3 patterns are detected, false otherwise. + * @throws IOException + */ + public static boolean checkN3Format(InputStream is) throws IOException { + return findPattern(N3_PATTERNS, '.', is); + } + + /** + * Checks if the stream contains the NQuads patterns. + * + * @param is input stream to be verified. + * @return true if N3 patterns are detected, false otherwise. + * @throws IOException + */ + public static boolean checkNQuadsFormat(InputStream is) throws IOException { + return findPattern(NQUADS_PATTERNS, '.', is); + } + + /** + * Checks if the stream contains Turtle triple patterns. + * + * @param is input stream to be verified. + * @return true if Turtle patterns are detected, false otherwise. + * @throws IOException + */ + public static boolean checkTurtleFormat(InputStream is) throws IOException { + String sample = extractDataSample(is, '.'); + RDFParser turtleParser = Rio.createParser(RDFFormat.TURTLE); + turtleParser.setDatatypeHandling(RDFParser.DatatypeHandling.VERIFY); + turtleParser.setStopAtFirstError(true); + turtleParser.setVerifyData(true); + ByteArrayInputStream bais = new ByteArrayInputStream( sample.getBytes() ); + try { + turtleParser.parse(bais, ""); + return true; + } catch (Exception e) { + return false; + } + } + + /* + * Checks if the stream contains a valid CSV pattern. + * + * @param is input stream to be verified. + * @return true if CSV patterns are detected, false otherwise. + * @throws IOException + * + public static boolean checkCSVFormat(InputStream is) throws IOException { + return CSVReaderBuilder.isCSV(is); + }*/ + //TODO add once CSV has been ported over from Any23 + + /** + * Tries to apply one of the given patterns on a sample of the input stream. + * + * @param patterns the patterns to apply. + * @param delimiterChar the delimiter of the sample. + * @param is the input stream to sample. + * @return true if a pattern has been applied, false otherwise. + * @throws IOException + */ + private static boolean findPattern(Pattern[] patterns, char delimiterChar, InputStream is) + throws IOException { + String sample = extractDataSample(is, delimiterChar); + for(Pattern pattern : patterns) { + if(pattern.matcher(sample).find()) { + return true; + } + } + return false; + } + + /** + * Extracts a sample data from the input stream, from the current + * mark to the first breakChar char. + * + * @param is the input stream to sample. + * @param breakChar the char to break to sample. + * @return the sample string. + * @throws IOException if an error occurs during sampling. + */ + private static String extractDataSample(InputStream is, char breakChar) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + StringBuilder sb = new StringBuilder(); + final int MAX_SIZE = 1024 * 2; + int c; + boolean insideBlock = false; + int read = 0; + br.mark(MAX_SIZE); + try { + while ((c = br.read()) != -1) { + read++; + if (read > MAX_SIZE) { + break; + } + if ('<' == c) { + insideBlock = true; + } else if ('>' == c) { + insideBlock = false; + } else if ('"' == c) { + insideBlock = !insideBlock; + } + sb.append((char) c); + if (!insideBlock && breakChar == c) { + break; + } + } + } finally { + is.reset(); + br.reset(); + } + return sb.toString(); + } + +} Index: tika-core/src/main/java/org/apache/tika/detect/EmptyDetector.java =================================================================== --- tika-core/src/main/java/org/apache/tika/detect/EmptyDetector.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/detect/EmptyDetector.java (working copy) @@ -28,6 +28,11 @@ public class EmptyDetector implements Detector { /** + * Generated serial ID + */ + private static final long serialVersionUID = -2739938917225728418L; + + /** * Singleton instance of this class. */ public static final EmptyDetector INSTANCE = new EmptyDetector(); Index: tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java =================================================================== --- tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java (working copy) @@ -27,6 +27,8 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.purifier.Purifier; +import org.apache.tika.mime.purifier.WhiteSpacesPurifier; /** * Content type detection based on magic bytes, i.e. type-specific patterns @@ -40,6 +42,11 @@ */ public class MagicDetector implements Detector { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -8869394154590907300L; + private static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); public static MagicDetector parse( @@ -354,6 +361,16 @@ return MediaType.OCTET_STREAM; } + //Purify the InputStream with the 'default' + //WhiteSpacePurifier. In the future we could make + //this configurable. + Purifier purifier = new WhiteSpacesPurifier(); + try { + purifier.purify(input); + } catch (IOException e) { + throw new RuntimeException("Error while purifying the provided InputStream", e); + } + input.mark(offsetRangeEnd + length); try { int offset = 0; Index: tika-core/src/main/java/org/apache/tika/detect/NameDetector.java =================================================================== --- tika-core/src/main/java/org/apache/tika/detect/NameDetector.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/detect/NameDetector.java (working copy) @@ -42,6 +42,11 @@ public class NameDetector implements Detector { /** + * Generated serial ID + */ + private static final long serialVersionUID = 2076296572544812146L; + + /** * The regular expression patterns used for type detection. */ private final Map patterns; Index: tika-core/src/main/java/org/apache/tika/detect/TypeDetector.java =================================================================== --- tika-core/src/main/java/org/apache/tika/detect/TypeDetector.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/detect/TypeDetector.java (working copy) @@ -31,6 +31,11 @@ public class TypeDetector implements Detector { /** + * Generated serial ID + */ + private static final long serialVersionUID = 6963170539158578784L; + + /** * Detects the content type of an input document based on a type hint * given in the input metadata. The CONTENT_TYPE attribute of the given * input metadata is expected to contain the type of the input document. Index: tika-core/src/main/java/org/apache/tika/extractor/ParserContainerExtractor.java =================================================================== --- tika-core/src/main/java/org/apache/tika/extractor/ParserContainerExtractor.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/extractor/ParserContainerExtractor.java (working copy) @@ -29,6 +29,8 @@ import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.purifier.Purifier; +import org.apache.tika.mime.purifier.WhiteSpacesPurifier; import org.apache.tika.parser.AbstractParser; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; @@ -53,15 +55,32 @@ private final Detector detector; + /** + * Default constructor using + * {@link org.apache.tika.config.TikaConfig#getDefaultConfig()} + */ public ParserContainerExtractor() { this(TikaConfig.getDefaultConfig()); } + /** + * Supplementary constructor which relies upon an existing + * {@link org.apache.tika.config.TikaConfig} instance. + * @param config the existing + * {@link org.apache.tika.config.TikaConfig} instance + */ public ParserContainerExtractor(TikaConfig config) { this(new AutoDetectParser(config), new DefaultDetector(config.getMimeRepository())); } + /** + * Supplementary constructor allowing us to specify the following + * @param parser the specific {@link org.apache.tika.parser.Parser} + * implementation + * @param detector the specific {@link org.apache.tika.detect.Detector} + * implementation. + */ public ParserContainerExtractor(Parser parser, Detector detector) { this.parser = parser; this.detector = detector; @@ -79,7 +98,15 @@ ParseContext context = new ParseContext(); context.set(Parser.class, new RecursiveParser(recurseExtractor, handler)); try { - parser.parse(stream, new DefaultHandler(), new Metadata(), context); + if(stream != null) { + try { + Purifier purifier = new WhiteSpacesPurifier(); + purifier.purify(stream); + } catch (IOException e) { + throw new RuntimeException("Error while purifying the provided input", e); + } + } + parser.parse(stream, new DefaultHandler(), new Metadata(), context); } catch (SAXException e) { throw new TikaException("Unexpected SAX exception", e); } @@ -87,6 +114,11 @@ private class RecursiveParser extends AbstractParser { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -6803100507239611231L; + private final ContainerExtractor extractor; private final EmbeddedResourceHandler handler; @@ -107,8 +139,10 @@ Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { TemporaryResources tmp = new TemporaryResources(); + try { TikaInputStream tis = TikaInputStream.get(stream, tmp); + // Figure out what we have to process String filename = metadata.get(Metadata.RESOURCE_NAME_KEY); Index: tika-core/src/main/java/org/apache/tika/metadata/PropertyTypeException.java =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/PropertyTypeException.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/metadata/PropertyTypeException.java (working copy) @@ -29,6 +29,11 @@ */ public final class PropertyTypeException extends IllegalArgumentException { + /** + * Generated serial ID + */ + private static final long serialVersionUID = 7164142889292576010L; + public PropertyTypeException(String msg) { super(msg); } Index: tika-core/src/main/java/org/apache/tika/language/LanguageProfilerBuilder.java =================================================================== --- tika-core/src/main/java/org/apache/tika/language/LanguageProfilerBuilder.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/language/LanguageProfilerBuilder.java (working copy) @@ -33,6 +33,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; + import org.apache.tika.exception.TikaException; /** @@ -692,6 +693,7 @@ this(16); } + @SuppressWarnings("unused") QuickStringBuffer(char[] value) { this.value = value; count = value.length; @@ -701,6 +703,7 @@ value = new char[length]; } + @SuppressWarnings("unused") QuickStringBuffer(String str) { this(str.length() + 16); append(str); Index: tika-core/src/main/java/org/apache/tika/language/LanguageProfile.java =================================================================== --- tika-core/src/main/java/org/apache/tika/language/LanguageProfile.java (revision 1556678) +++ tika-core/src/main/java/org/apache/tika/language/LanguageProfile.java (working copy) @@ -62,6 +62,7 @@ public LanguageProfile(String content, int length) { this(length); + @SuppressWarnings("resource") ProfilingWriter writer = new ProfilingWriter(this); char[] ch = content.toCharArray(); writer.write(ch, 0, ch.length); Index: tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml =================================================================== --- tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml (revision 1556678) +++ tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml (working copy) @@ -506,12 +506,24 @@ - + + + + RDF/XML <_comment>XML syntax for RDF graphs + + + + + + + + + @@ -634,6 +646,14 @@ + + + + + + + @@ -4609,7 +4629,16 @@ - + + + + + + + + + + @@ -4739,6 +4768,12 @@ + + + + + + @@ -5069,6 +5104,13 @@ + + + + + + + <_comment>Objective-C source code Index: tika-core/src/test/java/org/apache/tika/io/LookaheadInputStreamTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/io/LookaheadInputStreamTest.java (revision 1556678) +++ tika-core/src/test/java/org/apache/tika/io/LookaheadInputStreamTest.java (working copy) @@ -31,6 +31,7 @@ @Test public void testNullStream() throws IOException { + @SuppressWarnings("resource") InputStream lookahead = new LookaheadInputStream(null, 100); assertEquals(-1, lookahead.read()); } Index: tika-core/src/test/java/org/apache/tika/detect/Any23DetectorTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/detect/Any23DetectorTest.java (revision 0) +++ tika-core/src/test/java/org/apache/tika/detect/Any23DetectorTest.java (revision 0) @@ -0,0 +1,462 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import static org.junit.Assert.*; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.apache.tika.mime.purifier.WhiteSpacesPurifier; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.openrdf.rio.RDFFormat; + +public class Any23DetectorTest { + + private static final String PLAIN = "text/plain"; + private static final String HTML = "text/html"; + private static final String XML = "application/xml"; + private static final String TRIX = RDFFormat.TRIX.getDefaultMIMEType(); + private static final String XHTML = "application/xhtml+xml"; + private static final String RDFXML = RDFFormat.RDFXML.getDefaultMIMEType(); + private static final String TURTLE = RDFFormat.TURTLE.getDefaultMIMEType(); + private static final String N3 = RDFFormat.N3.getDefaultMIMEType(); + private static final String NQUADS = RDFFormat.NQUADS.getDefaultMIMEType(); + private static final String CSV = "text/csv"; + private static final String RSS = "application/rss+xml"; + private static final String ATOM = "application/atom+xml"; + + private Any23Detector detector; + + @Before + public void setUp() throws Exception { + detector = new Any23Detector(new WhiteSpacesPurifier()); + } + + @After + public void tearDown() throws Exception { + detector = null; + } + + @Test + public void testN3Detection() throws IOException { + assertN3Detection(" ."); + assertN3Detection("_:bnode1 _:bnode2 ."); + assertN3Detection(" \"x\" ."); + assertN3Detection(" \"x\"@it ."); + assertN3Detection(" \"x\"^^ ."); + assertN3Detection(" \"x\"^^xsd:integer ."); + + // Wrong N3 line '.' + assertN3DetectionFail("" + + " . " + ); + // NQuads is not mislead with N3. + assertN3DetectionFail( + " ." + ); + } + + @Test + public void testNQuadsDetection() throws IOException { + assertNQuadsDetection( + " ." + ); + assertNQuadsDetection( + "_:bnode1 _:bnode2 ." + ); + assertNQuadsDetection( + " \"x\" ." + ); + assertNQuadsDetection( + " \"x\"@it ." + ); + assertNQuadsDetection( + " \"xxx\"^^ ." + ); + assertNQuadsDetection( + " \"yyy\"^^xsd:datetime ." + ); + + // Wrong NQuads line. + assertNQuadsDetectionFail( + " \"x\"^^ . " + ); + // N3 is not mislead with NQuads. + assertNQuadsDetectionFail( + " ." + ); + } + + /* BEGIN: by content. */ + + @Test + public void testDetectRSS1ByContent() throws Exception { + detectMIMEtypeByContent(RDFXML, manifestRss1()); + } + + private List manifestRss1() { + return Arrays.asList("/application/rss1/test1"); + } + + @Test + public void testDetectRSS2ByContent() throws Exception { + detectMIMEtypeByContent(RSS, manifestRss2()); + } + + private List manifestRss2() { + return Arrays.asList("/application/rss2/index.html", "/application/rss2/rss2sample.xml", "/application/rss2/test1"); + } + + @Test + public void testDetectRDFN3ByContent() throws Exception { + detectMIMEtypeByContent(N3, manifestN3()); + } + + private List manifestN3() { + return Arrays.asList("/application/rdfn3/test1", "/application/rdfn3/test2", "/application/rdfn3/test3"); + } + + @Test + public void testDetectRDFNQuadsByContent() throws Exception { + detectMIMEtypeByContent(NQUADS, manifestNQuads()); + } + + private List manifestNQuads() { + return Arrays.asList("/application/nquads/test1.nq", "/application/nquads/test2.nq"); + } + + @Test + public void testDetectRDFXMLByContent() throws Exception { + detectMIMEtypeByContent(RDFXML, manifestRdfXml()); + } + + private List manifestRdfXml() { + return Arrays.asList("/application/rdfxml/error.rdf", "/application/rdfxml/foaf", "/application/rdfxml/physics.owl", "/application/rdfxml/test1", "/application/rdfxml/test2", "/application/rdfxml/test3"); + } + + @Test + public void testDetectTriXByContent() throws Exception { + detectMIMEtypeByContent(TRIX, manifestTrix()); + } + + private List manifestTrix() { + return Arrays.asList("/application/trix/test1.trx"); + } + + @Test + public void testDetectAtomByContent() throws Exception { + detectMIMEtypeByContent(ATOM, manifestAtom()); + } + + private List manifestAtom() { + return Arrays.asList("/application/atom/atom.xml"); + } + + @Test + public void testDetectHTMLByContent() throws Exception { + detectMIMEtypeByContent(HTML, manifestHtml()); + } + + private List manifestHtml() { + return Arrays.asList("/text/html/test1"); + } + + @Test + public void testDetectRDFaByContent() throws Exception { + detectMIMEtypeByContent(XHTML, manifestRdfa()); + } + + private List manifestRdfa() { + return Arrays.asList("/application/rdfa/false.test", "/application/rdfa/london-gazette.html", "/application/rdfa/mic.xhtml", "/application/rdfa/test1.html"); + } + + @Test + public void testDetectXHTMLByContent() throws Exception { + detectMIMEtypeByContent(XHTML, manifestXHtml()); + } + + private List manifestXHtml() { + return Arrays.asList("/application/xhtml/blank-file-header.xhtml", "/application/xhtml/index.html", "/application/xhtml/test1"); + } + + @Test + public void testDetectWSDLByContent() throws Exception { + detectMIMEtypeByContent("application/x-wsdl", manifestWsdl()); + } + + private List manifestWsdl() { + return Arrays.asList("/application/wsdl/error.wsdl", "/application/wsdl/test1"); + } + + @Test + public void testDetectZIPByContent() throws Exception { + detectMIMEtypeByContent("application/zip", manifestZip()); + } + + private List manifestZip() { + return Arrays.asList("/application/zip/4_entries.zip", "/application/zip/test1.zip", "/application/zip/test2"); + } + + @Test + public void testDetectCSVByContent() throws Exception { + detectMIMEtypeByContent(CSV, manifestCsv()); + } + + private List manifestCsv() { + return Arrays.asList("/org/apache/any23/extractor/csv/test-comma.csv", "/org/apache/any23/extractor/csv/test-semicolon.csv", "/org/apache/any23/extractor/csv/test-tab.csv", "/org/apache/any23/extractor/csv/test-type.csv"); + } + + /* END: by content. */ + + /* BEGIN: by content metadata. */ + + @Test + public void testDetectContentPlainByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(PLAIN, "text/plain"); + } + + @Test + public void testDetectTextRDFByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(RDFXML, "text/rdf"); + } + + @Test + public void testDetectTextN3ByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(N3, "text/rdf+n3"); + } + + @Test + public void testDetectTextNQuadsByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(NQUADS, "text/x-nquads"); + } + + @Test + public void testDetectTextTurtleByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(TURTLE, "text/turtle"); + } + + @Test + public void testDetectRDFXMLByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(RDFXML, "application/rdf+xml"); + } + + @Test + public void testDetectXMLByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(XML, "application/xml"); + } + + @Test + public void testDetectTriXByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(TRIX, "application/trix"); + } + + @Test + public void testDetectExtensionN3ByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(PLAIN, "text/plain"); + } + + @Test + public void testDetectXHTMLByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(XHTML, "application/xhtml+xml"); + } + + @Test + public void testDetectTextHTMLByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(HTML, "text/html"); + } + + @Test + public void testDetectTextPlainByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(PLAIN, "text/plain"); + } + + @Test + public void testDetectApplicationXMLByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(XML, "application/xml"); + } + + @Test + public void testDetectApplicationCSVByMeta() throws IOException { + detectMIMETypeByMimeTypeHint(CSV, "text/csv"); + } + + /* END: by content metadata. */ + + /* BEGIN: by content and name. */ + + @Test + public void testRDFXMLByContentAndName() throws Exception { + detectMIMETypeByContentAndName(RDFXML, manifestRdfXml()); + } + + @Test + public void testTriXByContentAndName() throws Exception { + detectMIMETypeByContentAndName(TRIX, manifestTrix()); + } + + @Test + public void testRSS1ByContentAndName() throws Exception { + detectMIMETypeByContentAndName(RDFXML, manifestRss1()); + } + + @Test + public void testRSS2ByContentAndName() throws Exception { + detectMIMETypeByContentAndName(RSS, manifestRss2()); + } + + @Test + public void testDetectRDFN3ByContentAndName() throws Exception { + detectMIMETypeByContentAndName(N3, manifestN3()); + } + + @Test + public void testDetectRDFNQuadsByContentAndName() throws Exception { + detectMIMETypeByContentAndName(NQUADS, manifestNQuads()); + } + + @Test + public void testAtomByContentAndName() throws Exception { + detectMIMETypeByContentAndName(ATOM, manifestAtom()); + } + + @Test + public void testHTMLByContentAndName() throws Exception { + detectMIMETypeByContentAndName(HTML, manifestHtml()); + } + + @Test + public void testXHTMLByContentAndName() throws Exception { + detectMIMETypeByContentAndName(XHTML, manifestXHtml()); + } + + @Test + public void testWSDLByContentAndName() throws Exception { + detectMIMETypeByContentAndName("application/x-wsdl", manifestWsdl()); + } + + @Test + public void testZipByContentAndName() throws Exception { + detectMIMETypeByContentAndName("application/zip", manifestZip()); + } + + @Test + public void testRDFaByContentAndName() throws Exception { + detectMIMETypeByContentAndName(XHTML, manifestRdfa()); + } + + @Test + public void testCSVByContentAndName() throws Exception { + detectMIMETypeByContentAndName(CSV, manifestCsv()); + } + + /* END: by content and name. */ + + private void assertN3Detection(String n3Exp) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream( n3Exp.getBytes() ); + assertTrue( Any23Detector.checkN3Format(bais) ); + } + + private void assertN3DetectionFail(String n3Exp) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream( n3Exp.getBytes() ); + assertFalse( Any23Detector.checkN3Format(bais) ); + } + + private void assertNQuadsDetection(String n4Exp) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream( n4Exp.getBytes() ); + assertTrue( Any23Detector.checkNQuadsFormat(bais) ); + } + + private void assertNQuadsDetectionFail(String n4Exp) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream( n4Exp.getBytes() ); + assertFalse( Any23Detector.checkNQuadsFormat(bais) ); + } + + /** + * Checks the detection of a specific MIME based on content analysis. + * + * @param expectedMimeType the expected mime type. + * @param testDir the target file. + * @throws IOException + */ + private void detectMIMEtypeByContent(String expectedMimeType, Collection manifest) + throws IOException { + String detectedMimeType; + for (String test : manifest) { + InputStream is = new BufferedInputStream(this.getClass().getResourceAsStream(test)); + detectedMimeType = detector.detect(is, null).toString(); + if (test.contains("error")) + Assert.assertNotSame(expectedMimeType, detectedMimeType); + else { + Assert.assertEquals( + String.format("Error in mimetype detection for file %s", test), + expectedMimeType, + detectedMimeType + ); + } + is.close(); + } + } + + /** + * Verifies the detection of a specific MIME based on content, filename and metadata MIME type. + * + * @param expectedMimeType + * @param contentTypeHeader + * @throws IOException + */ + private void detectMIMETypeByMimeTypeHint(String expectedMimeType, String contentTypeHeader) + throws IOException { + String detectedMimeType = detector.detect(expectedMimeType, MIMEType.parse(contentTypeHeader) + ).toString(); + Assert.assertEquals(expectedMimeType, detectedMimeType); + } + + /** + * Verifies the detection of a specific MIME based on content and filename. + * + * @param expectedMimeType + * @param testDir + * @throws IOException + */ + private void detectMIMETypeByContentAndName(String expectedMimeType, Collection manifest) throws IOException { + String detectedMimeType; + for (String test : manifest) { + InputStream is = new BufferedInputStream(this.getClass().getResourceAsStream(test)); + detectedMimeType = detector.guessMIMEType(test, is, null).toString(); + if (test.contains("error")) + Assert.assertNotSame(expectedMimeType, detectedMimeType); + else { + Assert.assertEquals( + String.format("Error while detecting mimetype in file %s", test), + expectedMimeType, + detectedMimeType + ); + } + is.close(); + } + } + +} Index: tika-core/src/test/java/org/apache/tika/language/LanguageProfilerBuilderTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/language/LanguageProfilerBuilderTest.java (revision 1556678) +++ tika-core/src/test/java/org/apache/tika/language/LanguageProfilerBuilderTest.java (working copy) @@ -81,6 +81,7 @@ InputStream stream = new FileInputStream(new File(profileName + "." + FILE_EXTENSION)); try { + @SuppressWarnings("resource") BufferedReader reader = new BufferedReader(new InputStreamReader( stream, encoding)); String line = reader.readLine(); Index: tika-core/src/test/java/org/apache/tika/mime/purifier/WhiteSpacesPurifierTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/mime/purifier/WhiteSpacesPurifierTest.java (revision 0) +++ tika-core/src/test/java/org/apache/tika/mime/purifier/WhiteSpacesPurifierTest.java (revision 0) @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tika.mime.purifier; + +import org.apache.tika.io.IOUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.*; + +/** + * Reference test case for + * {@link org.apache.tika.mime.purifier.WhiteSpacesPurifier}. + * The single resource used in this test is located in + * src/test/resources/org/apache/tika/mime/purifier/mimeblank-file-header.xhtml. + * + * @author Davide Palmisano ( dpalmisano@gmail.com ) + */ +public class WhiteSpacesPurifierTest { + + private Purifier purifier; + + @Before + public void setUp() { + this.purifier = new WhiteSpacesPurifier(); + } + + @After + public void tearDown() { + this.purifier = null; + } + + @Test + public void testPurification() throws IOException { + InputStream inputStream = + new BufferedInputStream(this.getClass().getResourceAsStream("blank-file-header.xhtml")); + this.purifier.purify(inputStream); + Assert.assertNotNull(inputStream); + Assert.assertTrue( + validatePurification( + IOUtils.toString(inputStream) + ) + ); + + } + + /** + * Checks if a {@link String} starts with a proper character. + * + * @param string + * @return + */ + private boolean validatePurification(String string) { + char firstChar = string.charAt(0); + return (firstChar != '\t') && (firstChar != '\n') && (firstChar != ' '); + } + +} Index: tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java (revision 1556678) +++ tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java (working copy) @@ -26,7 +26,6 @@ import org.apache.tika.config.TikaConfig; import org.apache.tika.metadata.Metadata; - import org.junit.Before; import org.junit.Test; @@ -74,6 +73,7 @@ testFile("image/cgm", "plotutils-bin-cgm-v3.cgm"); // test HTML detection of malformed file, previously identified as image/cgm (TIKA-1170) testFile("text/html", "test-malformed-header.html.bin"); + //TODO add tests for new MimeTypes in TIKA-1208 } @Test Index: tika-core/src/test/java/org/apache/tika/parser/CompositeParserTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/parser/CompositeParserTest.java (revision 1556678) +++ tika-core/src/test/java/org/apache/tika/parser/CompositeParserTest.java (working copy) @@ -41,16 +41,31 @@ @Test public void testFindDuplicateParsers() { Parser a = new EmptyParser() { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -8475080505932285169L; + public Set getSupportedTypes(ParseContext context) { return Collections.singleton(MediaType.TEXT_PLAIN); } }; Parser b = new EmptyParser() { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -4846333006512101154L; + public Set getSupportedTypes(ParseContext context) { return Collections.singleton(MediaType.TEXT_PLAIN); } }; Parser c = new EmptyParser() { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -8312856259921893617L; + public Set getSupportedTypes(ParseContext context) { return Collections.singleton(MediaType.OCTET_STREAM); } Index: tika-core/src/test/java/org/apache/tika/parser/DummyParser.java =================================================================== --- tika-core/src/test/java/org/apache/tika/parser/DummyParser.java (revision 1556678) +++ tika-core/src/test/java/org/apache/tika/parser/DummyParser.java (working copy) @@ -32,6 +32,10 @@ * A Dummy Parser for use with unit tests. */ public class DummyParser extends AbstractParser { + /** + * Generated serial ID + */ + private static final long serialVersionUID = -4197644127672051333L; private Set types; private Map metadata; private String xmlText; Index: tika-core/src/test/resources/org/apache/tika/mime/purifier/blank-file-header.xhtml =================================================================== --- tika-core/src/test/resources/org/apache/tika/mime/purifier/blank-file-header.xhtml (revision 0) +++ tika-core/src/test/resources/org/apache/tika/mime/purifier/blank-file-header.xhtml (revision 0) @@ -0,0 +1,13493 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Toy Story 3 Movie Reviews, Pictures - Rotten Tomatoes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + \ No newline at end of file Index: tika-core/pom.xml =================================================================== --- tika-core/pom.xml (revision 1556678) +++ tika-core/pom.xml (working copy) @@ -55,14 +55,33 @@ bndlib provided + + + + org.openrdf.sesame + sesame-rio-turtle + ${sesame.version} + compile + + + org.openrdf.sesame + sesame-rio-ntriples + ${sesame.version} + runtime + + + org.openrdf.sesame + sesame-rio-n3 + ${sesame.version} + runtime + junit junit test - 4.11 - + @@ -145,20 +164,20 @@ This is the core Apache Tika™ toolkit library from which all other modules inherit functionality. It also includes the core facades for the Tika API. - The Apache Software Foundation - http://www.apache.org + The Apache Software Foundation + http://www.apache.org - http://svn.apache.org/viewvc/tika/trunk/core - scm:svn:http://svn.apache.org/repos/asf/tika/trunk/core - scm:svn:https://svn.apache.org/repos/asf/tika/trunk/core + http://svn.apache.org/viewvc/tika/trunk/core + scm:svn:http://svn.apache.org/repos/asf/tika/trunk/core + scm:svn:https://svn.apache.org/repos/asf/tika/trunk/core - JIRA - https://issues.apache.org/jira/browse/TIKA + JIRA + https://issues.apache.org/jira/browse/TIKA - Jenkins - https://builds.apache.org/job/Tika-trunk/ + Jenkins + https://builds.apache.org/job/Tika-trunk/ Index: tika-parent/pom.xml =================================================================== --- tika-parent/pom.xml (revision 1556678) +++ tika-parent/pom.xml (working copy) @@ -252,6 +252,7 @@ 1.6 1.6 ${project.build.sourceEncoding} + 2.7.5