Index: tika-parsers/src/test/java/org/apache/tika/parser/jdbc/TestSQLLiteParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/test/java/org/apache/tika/parser/jdbc/TestSQLLiteParser.java (revision ) +++ tika-parsers/src/test/java/org/apache/tika/parser/jdbc/TestSQLLiteParser.java (revision ) @@ -0,0 +1,170 @@ +package org.apache.tika.parser.jdbc; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.TikaTest; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.parser.RecursiveParserWrapper; +import org.apache.tika.sax.BasicContentHandlerFactory; +import org.junit.Test; +import org.xml.sax.helpers.DefaultHandler; + +import java.io.InputStream; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TestSQLLiteParser extends TikaTest { +/* + private Connection getConnection(String dbFileName) throws Exception { + File testDirectory = new File(this.getClass().getResource("/test-documents").toURI()); + System.out.println(testDirectory.getAbsolutePath()); + File testDB = new File(testDirectory, dbFileName); + Connection c = null; + try { + Class.forName("org.sqlite.JDBC"); + c = DriverManager.getConnection("jdbc:sqlite:"+testDB.getAbsolutePath()); + } catch ( Exception e ) { + System.err.println( e.getClass().getName() + ": " + e.getMessage() ); + System.exit(0); + } + return c; + } + + @Test + public void testCreateDB() throws Exception { + Connection c = getConnection("testSQLLite3b.db"); + Statement st = c.createStatement(); + String sql = "DROP TABLE if exists my_table1"; + st.execute(sql); + sql = "CREATE TABLE my_table1 (" + + "INT_COL INT PRIMARY KEY, "+ + "FLOAT_COL FLOAT, " + + "DOUBLE_COL DOUBLE, " + + "CHAR_COL CHAR(30), "+ + "VARCHAR_COL VARCHAR(30), "+ + "BOOLEAN_COL BOOLEAN,"+ + "DATE_COL DATE,"+ + "TIME_STAMP_COL TIMESTAMP,"+ + "BYTES_COL BYTES" + + ")"; + st.execute(sql); + sql = "insert into my_table1 (INT_COL, FLOAT_COL, DOUBLE_COL, CHAR_COL, " + + "VARCHAR_COL, BOOLEAN_COL, DATE_COL, TIME_STAMP_COL, BYTES_COL) " + + "values (?,?,?,?,?,?,?,?,?)"; + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + java.util.Date d = f.parse("2015-01-03 15:17:03"); + System.out.println(d.getTime()); + long d1Long = 1420229823000L;// 2015-01-02 15:17:03 + long d2Long = 1420316223000L;// 2015-01-03 15:17:03 + PreparedStatement ps = c.prepareStatement(sql); + ps.setInt(1, 0); + ps.setFloat(2, 2.3f); + ps.setDouble(3, 2.4d); + ps.setString(4, "lorem"); + ps.setString(5, "普林斯顿大学"); + ps.setBoolean(6, true); + ps.setDate(7, new Date(d1Long)); + ps.setTimestamp(8, new Timestamp(d2Long)); +// ps.setClob(9, new StringReader(clobString)); + ps.setBytes(9, getByteArray(this.getClass().getResourceAsStream("/test-documents/testWORD_1img.docx")));//contains "quick brown fox" + ps.executeUpdate(); + ps.clearParameters(); + + ps.setInt(1, 1); + ps.setFloat(2, 4.6f); + ps.setDouble(3, 4.8d); + ps.setString(4, "dolor"); + ps.setString(5, "sit"); + ps.setBoolean(6, false); + ps.setDate(7, new Date(d1Long)); + ps.setTimestamp(8, new Timestamp(d2Long)); + //ps.setClob(9, new StringReader("consectetur adipiscing elit")); + ps.setBytes(9, getByteArray(this.getClass().getResourceAsStream("/test-documents/testWORD_1img.docx")));//contains "The end!" + + ps.executeUpdate(); + + //build table2 + sql = "DROP TABLE if exists my_table2"; + st.execute(sql); + + sql = "CREATE TABLE my_table2 (" + + "INT_COL2 INT PRIMARY KEY, "+ + "VARCHAR_COL2 VARCHAR(64))"; + st.execute(sql); + sql = "INSERT INTO my_table2 values(0,'sed do eiusmod tempor')"; + st.execute(sql); + sql = "INSERT INTO my_table2 values(1,'incididunt ut labore')"; + st.execute(sql); + + c.close(); + } + + private byte[] getByteArray(InputStream is) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + byte[] buff = new byte[1024]; + for (int bytesRead; (bytesRead = is.read(buff)) != -1;) { + bos.write(buff, 0, bytesRead); + } + return bos.toByteArray(); + } +*/ + + @Test + public void testBasic() throws Exception { + Parser p = new AutoDetectParser(); + ParseContext context = new ParseContext(); + context.set(Parser.class, p); + + InputStream is = getResourceAsStream("/test-documents/testSQLLite3b.db"); + Metadata metadata = new Metadata(); + metadata.set(Metadata.RESOURCE_NAME_KEY, "testSQLLite3b.db"); + XMLResult result = getXML(is, p, metadata); + System.out.println(result.xml); + } + + @Test + public void testRecursiveParserWrapper() throws Exception { + Parser p = new AutoDetectParser(); + + RecursiveParserWrapper wrapper = + new RecursiveParserWrapper(p, new BasicContentHandlerFactory( + BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1)); + InputStream is = getResourceAsStream("/test-documents/testSQLLite3b.db"); + Metadata metadata = new Metadata(); + metadata.set(Metadata.RESOURCE_NAME_KEY, "testSQLLite3b.db"); + wrapper.parse(is, new DefaultHandler(), metadata, new ParseContext()); + List metadataList = wrapper.getMetadata(); + System.out.println(metadataList.size()); + int i = 0; + for (Metadata m : metadataList) { + for (String n : m.names()) { + for (String v : m.getValues(n)) { + System.out.println(i + " : " + n + " : " + v); + } + } + System.out.println(""); + i++; + } + assertEquals(7, metadataList.size()); + } + +} Index: tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java (date 1421075775000) +++ tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java (revision ) @@ -17,6 +17,7 @@ package org.apache.tika.parser; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.PseudoInputStream; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -241,17 +242,24 @@ InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { + Parser parser = getParser(metadata, context); - TemporaryResources tmp = new TemporaryResources(); - try { - TikaInputStream taggedStream = TikaInputStream.get(stream, tmp); - TaggedContentHandler taggedHandler = - handler != null ? new TaggedContentHandler(handler) : null; - if (parser instanceof ParserDecorator){ - metadata.add("X-Parsed-By", ((ParserDecorator) parser).getWrappedParser().getClass().getName()); - } else { - metadata.add("X-Parsed-By", parser.getClass().getName()); - } + if (parser instanceof ParserDecorator){ + metadata.add("X-Parsed-By", ((ParserDecorator) parser).getWrappedParser().getClass().getName()); + } else { + metadata.add("X-Parsed-By", parser.getClass().getName()); + } + + if (stream instanceof PseudoInputStream) { + parser.parse(stream, handler, metadata, context); + return; + } + + TemporaryResources tmp = new TemporaryResources(); + try { + TikaInputStream taggedStream = TikaInputStream.get(stream, tmp); + TaggedContentHandler taggedHandler = + handler != null ? new TaggedContentHandler(handler) : null; try { parser.parse(taggedStream, taggedHandler, metadata, context); } catch (RuntimeException e) { Index: tika-parsers/pom.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/pom.xml (date 1421075775000) +++ tika-parsers/pom.xml (revision ) @@ -87,6 +87,7 @@ jmatio 1.0 + org.apache.james apache-mime4j-core @@ -166,6 +167,12 @@ asm-debug-all 4.1 + + org.xerial + sqlite-jdbc + 3.8.7 + + com.googlecode.mp4parser isoparser Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/JDBCResultSetInputStream.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/JDBCResultSetInputStream.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/JDBCResultSetInputStream.java (revision ) @@ -0,0 +1,34 @@ +package org.apache.tika.parser.jdbc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.io.PseudoInputStream; + +class JDBCResultSetInputStream extends PseudoInputStream { + + private final ResultSetTablePair resultSetTablePair; + + JDBCResultSetInputStream(ResultSetTablePair pair) { + this.resultSetTablePair = pair; + } + + @Override + public ResultSetTablePair getPayLoad() { + return resultSetTablePair; + } + +} Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3Parser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3Parser.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3Parser.java (revision ) @@ -0,0 +1,59 @@ +package org.apache.tika.parser.jdbc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.AbstractParser; +import org.apache.tika.parser.ParseContext; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Set; + +public class SQLite3Parser extends AbstractParser { + /** Serial version UID */ + private static final long serialVersionUID = -752276948656079347L; + + private static final MediaType MEDIA_TYPE = MediaType.application("x-sqlite3"); + + private static final Set SUPPORTED_TYPES = + Collections.singleton(MEDIA_TYPE); + + @Override + public Set getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + @Override + public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { + SQLite3DBParser p = new SQLite3DBParser(); + try { + p.parse(stream, handler, metadata, context); + } finally { + try { + p.close(); + } catch (SQLException e) { + throw new TikaException(e.getMessage()); + } + } + } +} Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractSingleFileDBParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractSingleFileDBParser.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractSingleFileDBParser.java (revision ) @@ -0,0 +1,57 @@ +package org.apache.tika.parser.jdbc; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.io.IOUtils; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; + +/** + * Convenience class that overrides getJDBCDBString to get a File from + * the InputStream via TikaInputStream. + */ +abstract class AbstractSingleFileDBParser extends AbstractDBParser { + + private TikaInputStream tis = null; + @Override + protected String getJDBCDBString(InputStream is, Metadata metadata, ParseContext context) throws IOException { + File dbFile = null; + tis = TikaInputStream.get(is); + dbFile = tis.getFile(); + + return dbFile.getAbsolutePath(); + } + + @Override + protected void close() throws SQLException,IOException { + //close the general db connection + try { + super.close(); + } finally { + //now close the tis + IOUtils.closeQuietly(tis); + } + } + +} Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3DBParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3DBParser.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3DBParser.java (revision ) @@ -0,0 +1,104 @@ +package org.apache.tika.parser.jdbc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.sqlite.SQLiteConfig; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * This is the implementation of the db parser for SQLite. + */ +class SQLite3DBParser extends AbstractSingleFileDBParser { + + final static MediaType SQLLITE_TABLE_MIME_TYPE = MediaType.parse("application/x-tika-sqlite3-table"); + + @Override + public Set getSupportedTypes(ParseContext context) { + return null; + } + + @Override + protected Connection getConnection(InputStream stream, Metadata metadata, ParseContext context) throws TikaException, IOException { + String connectionString = buildConnectionString(stream, metadata, context); + + Connection connection = null; + try { + Class.forName(getJDBCClassName()); + } catch (ClassNotFoundException e) { + throw new TikaException(e.getMessage()); + } + try{ + SQLiteConfig config = new SQLiteConfig(); + config.setReadOnly(true); + connection = config.createConnection(connectionString); + } catch (SQLException e) { + throw new IOException(e.getMessage()); + } + return connection; + } + + @Override + protected String getJDBCClassName() { + return "org.sqlite.JDBC"; + } + + @Override + protected String getJDBCProtocolString() { + return "jdbc:sqlite"; + } + + @Override + protected List getTableNames(Statement statement, ParseContext context) throws SQLException { + String sql = "SELECT name FROM sqlite_master WHERE type='table'"; + ResultSet rs = statement.executeQuery(sql); + + List tableNames = new LinkedList(); + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + return tableNames; + } + + @Override + protected List getPreSelectConfigureSQLCalls() { + return Collections.EMPTY_LIST; + } + + @Override + protected List getPreCloseConnectionSQLCalls() { + return Collections.EMPTY_LIST; + } + + @Override + protected MediaType getTableMediaType() { + return SQLLITE_TABLE_MIME_TYPE; + } +} Index: tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java (date 1421075775000) +++ tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java (revision ) @@ -16,13 +16,9 @@ */ package org.apache.tika.extractor; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; - import org.apache.tika.exception.TikaException; import org.apache.tika.io.CloseShieldInputStream; +import org.apache.tika.io.PseudoInputStream; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -35,6 +31,11 @@ import org.xml.sax.SAXException; import org.xml.sax.helpers.AttributesImpl; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; + import static org.apache.tika.sax.XHTMLContentHandler.XHTML; /** @@ -88,26 +89,37 @@ handler.characters(chars, 0, chars.length); handler.endElement(XHTML, "h1", "h1"); } - + if (stream instanceof PseudoInputStream) { + try { + DELEGATING_PARSER.parse( + stream, + new EmbeddedContentHandler(new BodyContentHandler(handler)), + metadata, context); + } catch (TikaException e) { + // TODO: can we log a warning somehow? + // Could not parse the entry, just skip the content + } + } else { - // Use the delegate parser to parse this entry - TemporaryResources tmp = new TemporaryResources(); - try { - final TikaInputStream newStream = TikaInputStream.get(new CloseShieldInputStream(stream), tmp); - if (stream instanceof TikaInputStream) { - final Object container = ((TikaInputStream) stream).getOpenContainer(); - if (container != null) { - newStream.setOpenContainer(container); - } - } - DELEGATING_PARSER.parse( - newStream, - new EmbeddedContentHandler(new BodyContentHandler(handler)), - metadata, context); - } catch (TikaException e) { - // TODO: can we log a warning somehow? - // Could not parse the entry, just skip the content - } finally { - tmp.close(); + // Use the delegate parser to parse this entry + TemporaryResources tmp = new TemporaryResources(); + try { + final TikaInputStream newStream = TikaInputStream.get(new CloseShieldInputStream(stream), tmp); + if (stream instanceof TikaInputStream) { + final Object container = ((TikaInputStream) stream).getOpenContainer(); + if (container != null) { + newStream.setOpenContainer(container); + } + } + DELEGATING_PARSER.parse( + newStream, + new EmbeddedContentHandler(new BodyContentHandler(handler)), + metadata, context); + } catch (TikaException e) { + // TODO: can we log a warning somehow? + // Could not parse the entry, just skip the content + } finally { + tmp.close(); + } } if(outputHtml) { Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractTableParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractTableParser.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractTableParser.java (revision ) @@ -0,0 +1,189 @@ +package org.apache.tika.parser.jdbc; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.io.IOUtils; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.JDBC; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AbstractParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.XHTMLContentHandler; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.sql.Clob; +import java.sql.Date; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; + +/** + * Abstract class that handles the basics of parsing a table. + */ +abstract class AbstractTableParser extends AbstractParser { + + int maxClobLength = 1000000; + + @Override + public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) + throws IOException, SAXException, TikaException { + if (! (stream instanceof JDBCResultSetInputStream)) { + throw new TikaException("InputStream for AbstractTableParser must be a JDBCResultSetInputStream"); + } + ResultSetTablePair pair = ((JDBCResultSetInputStream)stream).getPayLoad(); + Metadata m = new Metadata(); + m.set(JDBC.TABLE_NAME, pair.getTableName()); + XHTMLContentHandler xHandler = new XHTMLContentHandler(handler, m); + try { + ResultSet rs = pair.getResultSet(); + addResultSetMetadata(rs, metadata); + handleResultSet(pair.getTableName(), rs, xHandler, context); + } catch (SQLException e) { + throw new TikaException(e.getMessage()); + } + + } + + protected void addResultSetMetadata(ResultSet rs, Metadata metadata) throws SQLException { + ResultSetMetaData rsmd = rs.getMetaData(); + metadata.set(JDBC.COLUMN_COUNT, Integer.toString(rsmd.getColumnCount())); + for (int i = 1; i <= rsmd.getColumnCount(); i++) { + metadata.add(JDBC.COLUMN_NAME, rsmd.getColumnName(i)); + } + } + + private void handleResultSet(String tableName, ResultSet rs, XHTMLContentHandler handler, ParseContext context) throws SQLException, SAXException, IOException { + ResultSetMetaData rsmd = rs.getMetaData(); + int rows = 0; + handler.startElement("table", "name", tableName); + while (rs.next()) { + handler.startElement("tr"); + for (int i = 1; i <= rsmd.getColumnCount(); i++) { + handler.startElement("td"); + switch (rsmd.getColumnType(i)) { + case Types.BLOB : + handleBlob(tableName, rsmd.getColumnName(i), rows, rs, i, handler, context); + break; + case Types.CLOB : + handleClob(tableName, rsmd.getColumnName(i), rows, rs, i, handler, context); + break; + case Types.DATE : + handleDate(rs.getDate(i), handler); + break; + case Types.TIMESTAMP : + handleTimeStamp(rs.getTimestamp(i), handler); + break; + case Types.BOOLEAN : + handleBoolean(rs.getBoolean(i), handler); + break; + case Types.INTEGER : + handleInteger(rsmd.getColumnTypeName(i), rs, i, handler); + break; + case Types.FLOAT : + //this is necessary to handle rounding issues in presentation + handler.characters(Float.toString(rs.getFloat(i))); + break; + case Types.DOUBLE : + handler.characters(Double.toString(rs.getDouble(i))); + break; + default : + handler.characters(rs.getString(i)); + break; + } + handler.endElement("td"); + } + handler.endElement("tr"); + rows++; + } + handler.endElement("table"); + + } + + protected void handleInteger(String columnTypeName, ResultSet rs, int columnIndex, ContentHandler handler) throws SQLException, SAXException { + addAllCharacters(Integer.toString(rs.getInt(columnIndex)), handler); + } + + private void handleBoolean(boolean aBoolean, ContentHandler handler) throws SAXException { + addAllCharacters(Boolean.toString(aBoolean), handler); + } + + + protected void handleClob(String tableName, String fieldName, int rowNum, + ResultSet resultSet, int columnIndex, + ContentHandler handler, ParseContext context) throws SQLException, IOException, SAXException { + Clob clob = resultSet.getClob(columnIndex); + boolean truncated = clob.length() > Integer.MAX_VALUE || clob.length() > maxClobLength; + + int readSize = (clob.length() < maxClobLength ? (int)clob.length() : maxClobLength); + Metadata m = new Metadata(); + m.set("JDBC_TABLE_NAME", tableName); + m.set("JDBC_FIELD_NAME", fieldName); + m.set("JDBC_ROW_NUM", Integer.toString(rowNum)); + m.set("JDBC_CLOB", "true"); + m.set("JDCB_CLOB_LENGTH", Long.toString(clob.length())); + m.set("JDBC_CLOB_TRUNCATED", Boolean.toString(truncated)); + m.set(Metadata.CONTENT_TYPE, "text/plain; charset=UTF-8"); + m.set(Metadata.CONTENT_LENGTH, Integer.toString(readSize)); + + //is there a more efficient way to go from a Reader to an InputStream? + String s = clob.getSubString(0, readSize); + EmbeddedDocumentExtractor ex = AbstractDBParser.getEmbeddedDocumentExtractor(context); + ex.parseEmbedded(new ByteArrayInputStream(s.getBytes("UTF-8")), handler, m, false); + } + + protected void handleBlob(String tableName, String fieldName, int rowNum, ResultSet resultSet, int columnIndex, + ContentHandler handler, ParseContext context) throws SQLException, IOException, SAXException { + Metadata m = new Metadata(); + m.set("JDBC_TABLE_NAME", tableName); + m.set("JDBC_FIELD_NAME", fieldName); + m.set("JDBC_ROW_NUM", Integer.toString(rowNum)); + m.set("JDBC_BLOB", "true"); + EmbeddedDocumentExtractor ex = AbstractDBParser.getEmbeddedDocumentExtractor(context); + InputStream is = getInputStreamFromBlob(resultSet, columnIndex, m); + try { + ex.parseEmbedded(is, handler, m, false); + } finally { + IOUtils.closeQuietly(is); + } + } + + protected InputStream getInputStreamFromBlob(ResultSet resultSet, int columnIndex, Metadata metadata) throws SQLException { + return TikaInputStream.get(resultSet.getBlob(columnIndex), metadata); + } + + protected void handleDate(Date date, ContentHandler handler) throws SAXException { + addAllCharacters(date.toString(), handler); + } + + protected void handleTimeStamp(Timestamp timestamp, ContentHandler handler) throws SAXException { + addAllCharacters(timestamp.toString(), handler); + } + + protected void addAllCharacters(String s, ContentHandler handler) throws SAXException { + char[] chars = s.toCharArray(); + handler.characters(chars, 0, chars.length); + } +} Index: tika-core/src/main/java/org/apache/tika/metadata/JDBC.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/metadata/JDBC.java (revision ) +++ tika-core/src/main/java/org/apache/tika/metadata/JDBC.java (revision ) @@ -0,0 +1,25 @@ +package org.apache.tika.metadata; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public interface JDBC { + final static String PREFIX = "JDBC"+Metadata.NAMESPACE_PREFIX_DELIMITER; + + Property TABLE_NAME = Property.externalTextBag(PREFIX+"TABLE_NAME"); + Property COLUMN_COUNT = Property.externalText(PREFIX+"COLUMN_COUNT"); + Property COLUMN_NAME = Property.externalTextBag(PREFIX+"COLUMN_NAME"); +} Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/ResultSetTablePair.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/ResultSetTablePair.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/ResultSetTablePair.java (revision ) @@ -0,0 +1,42 @@ +package org.apache.tika.parser.jdbc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.sql.ResultSet; + +/** + * Simple class to hold a ResultSet and its corresponding table name + */ +public class ResultSetTablePair { + private final ResultSet resultSet; + private final String tableName; + + ResultSetTablePair(ResultSet resultSet, String tableName) { + this.resultSet = resultSet; + this.tableName = tableName; + } + + ResultSet getResultSet() { + return resultSet; + } + + String getTableName() { + return tableName; + } + + +} Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3TableParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3TableParser.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/SQLite3TableParser.java (revision ) @@ -0,0 +1,108 @@ +package org.apache.tika.parser.jdbc; + +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.Set; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Concrete class for SQLLite table parsing. This overrides + * column type handling from AbstractTableParser. + *

+ * This class is not designed to be thread safe (because of DateFormats)! + * Need to call a new instance for each db, as AbstractDBParser does. + *

+ * For now, this silently skips cells of type CLOB, because the jdbc connector + * does not currently support them. + */ +public class SQLite3TableParser extends AbstractTableParser { + + private static final MediaType MEDIA_TYPE = SQLite3DBParser.SQLLITE_TABLE_MIME_TYPE; + + private static final Set SUPPORTED_TYPES = + Collections.singleton(MEDIA_TYPE); + + + DateFormat timeStampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + + @Override + public Set getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + @Override + protected void handleClob(String tableName, String fieldName, int rowNum, + ResultSet resultSet, int columnIndex, + ContentHandler handler, ParseContext context) throws SQLException, IOException, SAXException { + //no-op for now. + } + + /** + * The jdbc connection to SqlLite does not yet implement blob, have to getBytes(). + * + * @param resultSet resultSet + * @param columnIndex columnIndex for blob + * @return + * @throws SQLException + */ + @Override + protected InputStream getInputStreamFromBlob(ResultSet resultSet, int columnIndex, Metadata m) throws SQLException { + return TikaInputStream.get(resultSet.getBytes(columnIndex), m); + } + + @Override + protected void handleTimeStamp(Timestamp timestamp, ContentHandler handler) throws SAXException { + Date d = new Date(timestamp.getTime()); + addAllCharacters(timeStampFormat.format(d),handler); + } + + @Override + protected void handleInteger(String columnTypeName, ResultSet rs, int columnIndex, ContentHandler handler) throws SQLException, SAXException { + //Yes, sqllite's jdbc connector is this crazy as of this writing. + //A timestamp is stored as a column of type Integer, but the columnTypeName is TIMESTAMP, and the + //value is a string representing a Long. + if (columnTypeName.equals("TIMESTAMP")) { + addAllCharacters(parseDateFromLongString(rs.getString(columnIndex)), handler); + } else { + addAllCharacters(Integer.toString(rs.getInt(columnIndex)), handler); + } + + } + + private String parseDateFromLongString(String longString) throws SAXException { + java.sql.Date d = new java.sql.Date(Long.parseLong(longString)); + return dateFormat.format(d); + + } +} Index: tika-core/src/main/java/org/apache/tika/io/PseudoInputStream.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/io/PseudoInputStream.java (revision ) +++ tika-core/src/main/java/org/apache/tika/io/PseudoInputStream.java (revision ) @@ -0,0 +1,50 @@ +package org.apache.tika.io; + +import java.io.IOException; +import java.io.InputStream; + +/** + * This is a wrapper around InputStream that allows the passing of an + * object to a parser instead of an actual InputStream. + *

+ * This was initially added for passing a ResultSet to a jdbc TableParser. + *

+ * Note that when you use a PseudoInputStream, make sure to specify the + * CONTENT_TYPE in the Metadata object if passing to AutoDetectParser. + * + * @param The payload object that should be retrieved by the parser + */ +public abstract class PseudoInputStream extends InputStream { + + /** + * + * @return the object that is wrapped in the InputStream + */ + public abstract T getPayLoad(); + + /** + * This should never actually be called. + * + * @return -1 always + * @throws IOException + */ + @Override + public int read() throws IOException { + return -1; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void mark(int readLimit) { + //no-op + } + + @Override + public void reset() { + //no-op + } +} Index: tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser (date 1421075775000) +++ tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser (revision ) @@ -56,3 +56,5 @@ org.apache.tika.parser.mat.MatParser org.apache.tika.parser.ocr.TesseractOCRParser org.apache.tika.parser.gdal.GDALParser +org.apache.tika.parser.jdbc.SQLite3TableParser +org.apache.tika.parser.jdbc.SQLite3Parser Index: tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java (date 1421075775000) +++ tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java (revision ) @@ -16,13 +16,11 @@ */ package org.apache.tika.parser; -import java.io.IOException; -import java.io.InputStream; - import org.apache.tika.config.TikaConfig; import org.apache.tika.detect.DefaultDetector; import org.apache.tika.detect.Detector; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.PseudoInputStream; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -32,6 +30,9 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; +import java.io.IOException; +import java.io.InputStream; + public class AutoDetectParser extends CompositeParser { /** Serial version UID */ @@ -104,27 +105,34 @@ InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { + + if (stream instanceof PseudoInputStream) { + //The CONTENT_TYPE should already have been set + //no need to detect. Do not wrap a pseudoStream + super.parse(stream, handler, metadata, context); + } else { - TemporaryResources tmp = new TemporaryResources(); - try { - TikaInputStream tis = TikaInputStream.get(stream, tmp); + TemporaryResources tmp = new TemporaryResources(); + try { + TikaInputStream tis = TikaInputStream.get(stream, tmp); - // Automatically detect the MIME type of the document - MediaType type = detector.detect(tis, metadata); - metadata.set(Metadata.CONTENT_TYPE, type.toString()); + // Automatically detect the MIME type of the document + MediaType type = detector.detect(tis, metadata); + metadata.set(Metadata.CONTENT_TYPE, type.toString()); - // TIKA-216: Zip bomb prevention - SecureContentHandler sch = - handler != null ? new SecureContentHandler(handler, tis) : null; - try { - // Parse the document - super.parse(tis, sch, metadata, context); - } catch (SAXException e) { - // Convert zip bomb exceptions to TikaExceptions - sch.throwIfCauseOf(e); - throw e; - } - } finally { - tmp.dispose(); + // TIKA-216: Zip bomb prevention + SecureContentHandler sch = + handler != null ? new SecureContentHandler(handler, tis) : null; + try { + // Parse the document + super.parse(tis, sch, metadata, context); + } catch (SAXException e) { + // Convert zip bomb exceptions to TikaExceptions + sch.throwIfCauseOf(e); + throw e; + } + } finally { + tmp.dispose(); + } } } Index: tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractDBParser.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractDBParser.java (revision ) +++ tika-parsers/src/main/java/org/apache/tika/parser/jdbc/AbstractDBParser.java (revision ) @@ -0,0 +1,178 @@ +package org.apache.tika.parser.jdbc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor; +import org.apache.tika.metadata.JDBC; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.AbstractParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.EmbeddedContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Set; + +abstract class AbstractDBParser extends AbstractParser { + + private Connection connection; + + @Override + public Set getSupportedTypes(ParseContext context) { + return null; + } + + @Override + public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { + connection = getConnection(stream, metadata, context); + XHTMLContentHandler xHandler = null; + try { + Statement st = connection.createStatement(); + for (String sql : getPreSelectConfigureSQLCalls()) { + st.execute(sql); + } + List tableNames = getTableNames(st, context); + for (String tableName : tableNames) { + //parent metadata + metadata.add(JDBC.TABLE_NAME, tableName); + } + xHandler = new XHTMLContentHandler(handler, metadata); + xHandler.startDocument(); + for (String tableName : tableNames) { + String sql = "SELECT * FROM "+tableName; + ResultSet rs = st.executeQuery(sql); + InputStream jdbcResultSetInputStream = new JDBCResultSetInputStream(new ResultSetTablePair(rs, tableName)); + EmbeddedDocumentExtractor ex = AbstractDBParser.getEmbeddedDocumentExtractor(context); + Metadata m = new Metadata(); + //child metadata + m.set(JDBC.TABLE_NAME, tableName); + m.set(Metadata.CONTENT_TYPE, getTableMediaType().toString()); + ex.parseEmbedded(jdbcResultSetInputStream, new EmbeddedContentHandler(xHandler), m, true); + } + } catch (SQLException e) { + throw new TikaException(e.getMessage()); + } finally { + if (xHandler != null) { + xHandler.endDocument(); + } + } + } + + protected static EmbeddedDocumentExtractor getEmbeddedDocumentExtractor(ParseContext context) { + return context.get(EmbeddedDocumentExtractor.class, + new ParsingEmbeddedDocumentExtractor(context)); + } + + protected void close() throws SQLException, IOException { + List closingCalls = getPreCloseConnectionSQLCalls(); + if (closingCalls != null && closingCalls.size() > 0) { + Statement st = connection.createStatement(); + for (String sql : getPreCloseConnectionSQLCalls()) { + st.execute(sql); + } + } + connection.close(); + } + + protected Connection getConnection(InputStream stream, Metadata metadata, ParseContext context) throws IOException, TikaException { + String connectionString = buildConnectionString(stream, metadata, context); + + Connection connection = null; + try { + Class.forName(getJDBCClassName()); + } catch (ClassNotFoundException e) { + throw new TikaException(e.getMessage()); + } + try{ + connection = DriverManager.getConnection(connectionString); + } catch (SQLException e) { + throw new TikaException(e.getMessage()); + } + return connection; + } + + protected String buildConnectionString(InputStream stream, Metadata metadata, ParseContext context) throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append(getJDBCProtocolString()).append(":"); + sb.append(getJDBCDBString(stream, metadata, context)); + //TODO: add other properties: user name, password? + return sb.toString(); + } + + /** + * JDBC class name, e.g. org.sqlite.JDBC + * @return jdbc class name + */ + abstract protected String getJDBCClassName(); + + /** + * Protocol string, e.g. jdbc:sqlite. + * @return protocol string + */ + abstract protected String getJDBCProtocolString(); + + /** + * Returns database resource string for use in the connection string. + * E.g. the filename for single file-based databases, or the database name + * @param is inputstream + * @param metadata metadata + * @param context context + * @return database resource string + * @throws IOException + */ + abstract protected String getJDBCDBString(InputStream is, Metadata metadata, ParseContext context) throws IOException; + + /** + * Returns the names of the tables to process + * @param statement Statement to use to make the sql call(s) to get the names of the tables + * @return list of table names to process + * @throws SQLException + */ + abstract protected List getTableNames(Statement statement, ParseContext context) throws SQLException; + + /** + * Returns list of sql calls to make immediately after making the connection, but before + * processing each table. Use this to set any optimization features of the database, e.g. + * limit number of rows to hold in memory in result set. + * @return + */ + abstract protected List getPreSelectConfigureSQLCalls(); + + /** + * @return List of sql calls to make before closing the connection + */ + abstract protected List getPreCloseConnectionSQLCalls(); + + /** + * Returns the special tika-internal MediaType to be used by the embedded parser to determine + * which table parser to use for each table + * @return special tika-internal MediaType for this class's table + */ + abstract protected MediaType getTableMediaType(); +}