Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 1658107) +++ CHANGES.txt (working copy) @@ -1,5 +1,8 @@ Release 1.8 - Current Development + * A basic wrapper around the UNIX file command was + added to extract Strings. (TIKA-1541) + * Add test files and detection mechanism for Gridded Binary (GRIB) files. (TIKA-1539) Index: tika-parsers/src/main/java/org/apache/tika/parser/strings/FileConfig.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/strings/FileConfig.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/strings/FileConfig.java (working copy) @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +import java.io.Serializable; + +/** + * Configuration for the "file" (or file-alternative) command. + * + */ +public class FileConfig implements Serializable { + /** + * Serial version UID + */ + private static final long serialVersionUID = 5712655467296441314L; + + private String filePath = ""; + + private boolean mimetype = false; + + /** + * Default constructor. + */ + public FileConfig() { + // TODO Loads properties from InputStream. + } + + /** + * Returns the "file" installation folder. + * + * @return the "file" installation folder. + */ + public String getFilePath() { + return filePath; + } + + /** + * Sets the "file" installation folder. + * + * @param path + * the "file" installation folder. + */ + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + /** + * Returns {@code true} if the mime option is enabled. + * + * @return {@code true} if the mime option is enabled, {@code} otherwise. + */ + public boolean isMimetype() { + return mimetype; + } + + /** + * Sets the mime option. If {@code true}, it causes the file command to + * output mime type strings rather than the more traditional human readable + * ones. + * + * @param mimetype + */ + public void setMimetype(boolean mimetype) { + this.mimetype = mimetype; + } +} Index: tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsConfig.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsConfig.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsConfig.java (working copy) @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +import java.io.File; +import java.io.Serializable; + +/** + * Configuration for the "strings" (or strings-alternative) command. + * + */ +public class StringsConfig implements Serializable { + /** + * Serial version UID + */ + private static final long serialVersionUID = -1465227101645003594L; + + private String stringsPath = ""; + + // Minimum sequence length (characters) to print + private int minLength = 4; + + // Character encoding of the strings that are to be found + private StringsEncoding encoding = StringsEncoding.SINGLE_7_BIT; + + // Maximum time (seconds) to wait for the strings process termination + private int timeout = 120; + + /** + * Default constructor. + */ + public StringsConfig() { + // TODO Loads properties from InputStream. + } + + /** + * Returns the "strings" installation folder. + * + * @return the "strings" installation folder. + */ + public String getStringsPath() { + return this.stringsPath; + } + + /** + * Returns the minimum sequence length (characters) to print. + * + * @return the minimum sequence length (characters) to print. + */ + public int getMinLength() { + return this.minLength; + } + + /** + * Returns the character encoding of the strings that are to be found. + * + * @return {@see StringsEncoding} enum that represents the character encoding of the strings that are to be found. + */ + public StringsEncoding getEncoding() { + return this.encoding; + } + + /** + * Returns the maximum time (in seconds) to wait for the "strings" command + * to terminate. + * + * @return the maximum time (in seconds) to wait for the "strings" command + * to terminate. + */ + public int getTimeout() { + return this.timeout; + } + + /** + * Sets the "strings" installation folder. + * + * @param path the "strings" installation folder. + */ + public void setStringsPath(String path) { + char lastChar = path.charAt(path.length() - 1); + + if (lastChar != File.separatorChar) { + path += File.separatorChar; + } + this.stringsPath = path; + } + + /** + * Sets the minimum sequence length (characters) to print. + * + * @param minLength the minimum sequence length (characters) to print. + */ + public void setMinLength(int minLength) { + this.minLength = minLength; + } + + /** + * Sets the character encoding of the strings that are to be found. + * + * @param encoding {@see StringsEncoding} enum that represents the character encoding of the strings that are to be found. + */ + public void setEncodings(StringsEncoding encoding) { + this.encoding = encoding; + } + + /** + * Sets the maximum time (in seconds) to wait for the "strings" command to terminate. + * @param timeout the maximum time (in seconds) to wait for the "strings" command to terminate. + */ + public void setTimeout(int timeout) { + this.timeout = timeout; + } +} Index: tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsEncoding.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsEncoding.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsEncoding.java (working copy) @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +/** + * Character encoding of the strings that are to be found using the "strings" command. + * + */ +public enum StringsEncoding { + SINGLE_7_BIT('s', "single-7-bit-byte"), + SINGLE_8_BIT('S', "single-8-bit-byte"), + BIGENDIAN_16_BIT('b', "16-bit bigendian"), + LITTLEENDIAN_16_BIT('l', "16-bit littleendian"), + BIGENDIAN_32_BIT('B', "32-bit bigendian"), + LITTLEENDIAN_32_BIT('L', "32-bit littleendian"); + + private char value; + + private String encoding; + + private StringsEncoding(char value, String encoding) { + this.value = value; + this.encoding = encoding; + } + + public char get() { + return value; + } + + @Override + public String toString() { + return encoding; + } +} Index: tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsParser.java =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsParser.java (revision 0) +++ tika-parsers/src/main/java/org/apache/tika/parser/strings/StringsParser.java (working copy) @@ -0,0 +1,310 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.AbstractParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.external.ExternalParser; +import org.apache.tika.sax.XHTMLContentHandler; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +/** + * Parser that uses the "strings" (or strings-alternative) command to find the + * printable strings in a object, or other binary, file + * (application/octet-stream). Useful as "best-effort" parser for files detected + * as application/octet-stream. + * + * @author gtotaro + * + */ +public class StringsParser extends AbstractParser { + /** + * Serial version UID + */ + private static final long serialVersionUID = 802566634661575025L; + + private static final Set SUPPORTED_TYPES = Collections + .singleton(MediaType.OCTET_STREAM); + + private static final StringsConfig DEFAULT_STRINGS_CONFIG = new StringsConfig(); + + private static final FileConfig DEFAULT_FILE_CONFIG = new FileConfig(); + + @Override + public Set getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + @Override + public void parse(InputStream stream, ContentHandler handler, + Metadata metadata, ParseContext context) throws IOException, + SAXException, TikaException { + StringsConfig stringsConfig = context.get(StringsConfig.class, DEFAULT_STRINGS_CONFIG); + FileConfig fileConfig = context.get(FileConfig.class, DEFAULT_FILE_CONFIG); + + if (!hasStrings(stringsConfig)) { + return; + } + + TikaInputStream tis = TikaInputStream.get(stream); + File input = tis.getFile(); + + // Metadata + metadata.set("strings:min-len", "" + stringsConfig.getMinLength()); + metadata.set("strings:encoding", stringsConfig.toString()); + metadata.set("strings:file_output", doFile(input, fileConfig)); + + int totalBytes = 0; + + // Content + XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); + + xhtml.startDocument(); + + totalBytes = doStrings(input, stringsConfig, xhtml); + + xhtml.endDocument(); + + // Metadata + metadata.set("strings:length", "" + totalBytes); + } + + /** + * Checks if the "strings" command is supported. + * + * @param config + * {@see StringsConfig} object used for testing the strings + * command. + * @return Returns returns {@code true} if the strings command is supported. + */ + private boolean hasStrings(StringsConfig config) { + String stringsProg = config.getStringsPath() + getStringsProg(); + + String[] checkCmd = { stringsProg, "--version" }; + + boolean hasStrings = ExternalParser.check(checkCmd); + + return hasStrings; + } + + /** + * Checks if the "file" command is supported. + * + * @param config + * @return + */ + private boolean hasFile(FileConfig config) { + String fileProg = config.getFilePath() + getFileProg(); + + String[] checkCmd = { fileProg, "--version" }; + + boolean hasFile = ExternalParser.check(checkCmd); + + return hasFile; + } + + /** + * Runs the "strings" command on the given file. + * + * @param input + * {@see File} object that represents the file to parse. + * @param config + * {@see StringsConfig} object including the strings + * configuration. + * @param xhtml + * {@see XHTMLContentHandler} object. + * @return the total number of bytes read using the strings command. + * @throws IOException + * if any I/O error occurs. + * @throws TikaException + * if the parsing process has been interrupted. + * @throws SAXException + */ + private int doStrings(File input, StringsConfig config, + XHTMLContentHandler xhtml) throws IOException, TikaException, + SAXException { + + // Builds the command array + ArrayList cmdList = new ArrayList(4); + cmdList.add(config.getStringsPath() + getStringsProg()); + cmdList.add("-n"); + cmdList.add("" + config.getMinLength());; + // encoding option is not supported by windows version + if (!getStringsProg().endsWith(".exe")) { + cmdList.add("-e"); + cmdList.add("" + config.getEncoding().get()); + } + cmdList.add(input.getPath()); + + String[] cmd = cmdList.toArray(new String[cmdList.size()]); + + ProcessBuilder pb = new ProcessBuilder(cmd); + final Process process = pb.start(); + + InputStream out = process.getInputStream(); + + FutureTask waitTask = new FutureTask( + new Callable() { + public Integer call() throws Exception { + return process.waitFor(); + } + }); + + Thread waitThread = new Thread(waitTask); + waitThread.start(); + + // Reads content printed out by "strings" command + int totalBytes = 0; + totalBytes = extractOutput(out, xhtml); + + try { + waitTask.get(config.getTimeout(), TimeUnit.SECONDS); + + } catch (InterruptedException ie) { + waitThread.interrupt(); + process.destroy(); + Thread.currentThread().interrupt(); + throw new TikaException(StringsParser.class.getName() + + " interrupted", ie); + + } catch (ExecutionException ee) { + // should not be thrown + + } catch (TimeoutException te) { + waitThread.interrupt(); + process.destroy(); + throw new TikaException(StringsParser.class.getName() + " timeout", + te); + } + + return totalBytes; + } + + /** + * Extracts ASCII strings using the "strings" command. + * + * @param stream + * {@see InputStream} object used for reading the binary file. + * @param xhtml + * {@see XHTMLContentHandler} object. + * @return the total number of bytes read using the "strings" command. + * @throws SAXException + * if the content element could not be written. + * @throws IOException + * if any I/O error occurs. + */ + private int extractOutput(InputStream stream, XHTMLContentHandler xhtml) + throws SAXException, IOException { + + char[] buffer = new char[1024]; + BufferedReader reader = null; + int totalBytes = 0; + + try { + reader = new BufferedReader(new InputStreamReader(stream)); + + int n = 0; + while ((n = reader.read(buffer)) != -1) { + if (n > 0) { + xhtml.characters(buffer, 0, n); + } + totalBytes += n; + } + + } finally { + reader.close(); + } + + return totalBytes; + } + + /** + * Runs the "file" command on the given file that aims at providing an + * alternative way to determine the file type. + * + * @param input + * {@see File} object that represents the file to detect. + * @return the file type provided by the "file" command using the "-b" + * option (it stands for "brief mode"). + * @throws IOException + * if any I/O error occurs. + */ + private String doFile(File input, FileConfig config) throws IOException { + if (!hasFile(config)) { + return null; + } + + // Builds the command array + ArrayList cmdList = new ArrayList(3); + cmdList.add(config.getFilePath() + getFileProg()); + cmdList.add("-b"); + if (config.isMimetype()) { + cmdList.add("-I"); + } + cmdList.add(input.getPath()); + + String[] cmd = cmdList.toArray(new String[cmdList.size()]); + + ProcessBuilder pb = new ProcessBuilder(cmd); + final Process process = pb.start(); + + InputStream out = process.getInputStream(); + + BufferedReader reader = null; + String fileOutput = null; + + try { + reader = new BufferedReader(new InputStreamReader(out)); + fileOutput = reader.readLine(); + + } catch (IOException ioe) { + // TODO + System.err + .println("An error occurred in reading output of the file command: " + + ioe.getMessage()); + } finally { + reader.close(); + } + + return fileOutput; + } + + + public static String getStringsProg() { + return System.getProperty("os.name").startsWith("Windows") ? "strings.exe" : "strings"; + } + + public static String getFileProg() { + return System.getProperty("os.name").startsWith("Windows") ? "file.exe" : "file"; + } +} Index: tika-parsers/src/test/java/org/apache/tika/parser/strings/FileConfigTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/strings/FileConfigTest.java (revision 0) +++ tika-parsers/src/test/java/org/apache/tika/parser/strings/FileConfigTest.java (working copy) @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class FileConfigTest { + + @Test + public void testNoConfig() { + FileConfig config = new FileConfig(); + assertEquals("Invalid default filePath value", "", config.getFilePath()); + assertEquals("Invalid default mime option value", false, config.isMimetype()); + } +} Index: tika-parsers/src/test/java/org/apache/tika/parser/strings/StringsConfigTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/strings/StringsConfigTest.java (revision 0) +++ tika-parsers/src/test/java/org/apache/tika/parser/strings/StringsConfigTest.java (working copy) @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class StringsConfigTest { + + @Test + public void testNoConfig() { + StringsConfig config = new StringsConfig(); + assertEquals("Invalid default filePath value", "", config.getStringsPath()); + assertEquals("Invalid default encoding value", StringsEncoding.SINGLE_7_BIT, config.getEncoding()); + assertEquals("Invalid default min-len value", 4, config.getMinLength()); + assertEquals("Invalid default timeout value", 120, config.getTimeout()); + } +} Index: tika-parsers/src/test/java/org/apache/tika/parser/strings/StringsParserTest.java =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/strings/StringsParserTest.java (revision 0) +++ tika-parsers/src/test/java/org/apache/tika/parser/strings/StringsParserTest.java (working copy) @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.strings; + +import static org.apache.tika.parser.strings.StringsParser.getStringsProg; +import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; + +import java.io.InputStream; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.parser.external.ExternalParser; +import org.apache.tika.sax.BodyContentHandler; +import org.junit.Test; +import org.xml.sax.ContentHandler; + +public class StringsParserTest { + public static boolean canRun() { + StringsConfig config = new StringsConfig(); + String[] checkCmd = {config.getStringsPath() + getStringsProg(), "--version"}; + boolean hasStrings = ExternalParser.check(checkCmd); + return hasStrings; + } + + @Test + public void testParse() throws Exception { + assumeTrue(canRun()); + + String resource = "/test-documents/testOCTET_header.dbase3"; + + String[] content = { "CLASSNO", "TITLE", "ITEMNO", "LISTNO", "LISTDATE" }; + + StringsConfig stringsConfig = new StringsConfig(); + //stringsConfig.setStringsPath("/usr/local/bin"); + FileConfig fileConfig = new FileConfig(); + + Parser parser = new StringsParser(); + ContentHandler handler = new BodyContentHandler(); + + ParseContext context = new ParseContext(); + context.set(StringsConfig.class, stringsConfig); + context.set(FileConfig.class, fileConfig); + + InputStream stream = StringsParserTest.class.getResourceAsStream(resource); + + try { + parser.parse(stream, handler, new Metadata(), context); + } finally { + stream.close(); + } + + System.out.println(handler.toString()); + + for (String word : content) { + assertTrue(handler.toString().contains(word)); + } + } +} Index: tika-parsers/src/test/resources/test-documents/testOCTET_header.dbase3 =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: tika-parsers/src/test/resources/test-documents/testOCTET_header.dbase3 =================================================================== --- tika-parsers/src/test/resources/test-documents/testOCTET_header.dbase3 (revision 1658107) +++ tika-parsers/src/test/resources/test-documents/testOCTET_header.dbase3 (working copy) Property changes on: tika-parsers/src/test/resources/test-documents/testOCTET_header.dbase3 ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +application/octet-stream \ No newline at end of property