Index: pom.xml =================================================================== --- pom.xml (revision 1598698) +++ pom.xml (working copy) @@ -49,6 +49,7 @@ tika-core tika-parsers tika-xmp + tika-serialization tika-app tika-bundle tika-server Index: tika-server/src/test/java/org/apache/tika/server/MetadataEPTest.java =================================================================== --- tika-server/src/test/java/org/apache/tika/server/MetadataEPTest.java (revision 1598698) +++ tika-server/src/test/java/org/apache/tika/server/MetadataEPTest.java (working copy) @@ -39,7 +39,8 @@ import org.apache.cxf.jaxrs.JAXRSServerFactoryBean; import org.apache.cxf.jaxrs.client.WebClient; import org.apache.tika.io.IOUtils; -import org.eclipse.jetty.util.ajax.JSON; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.serialization.JsonMetadata; import org.junit.Assert; import org.junit.Test; @@ -105,8 +106,7 @@ Assert.assertEquals(Status.OK.getStatusCode(), response.getStatus()); Reader reader = new InputStreamReader((InputStream) response.getEntity()); - Map metadata = (Map) JSON.parse(reader); - + Metadata metadata = JsonMetadata.fromJson(reader); assertNotNull(metadata.get("Author")); assertEquals("Maxim Valyanskiy", metadata.get("Author")); } @@ -129,7 +129,7 @@ Assert.assertEquals(Status.OK.getStatusCode(), response.getStatus()); Reader reader = new InputStreamReader((InputStream) response.getEntity()); - Map metadata = (Map) JSON.parse(reader); + Metadata metadata = JsonMetadata.fromJson(reader); assertNotNull(metadata.get("Author")); assertEquals("Maxim Valyanskiy", metadata.get("Author")); Index: tika-server/src/main/java/org/apache/tika/server/JSONMessageBodyWriter.java =================================================================== --- tika-server/src/main/java/org/apache/tika/server/JSONMessageBodyWriter.java (revision 1598698) +++ tika-server/src/main/java/org/apache/tika/server/JSONMessageBodyWriter.java (working copy) @@ -17,9 +17,9 @@ package org.apache.tika.server; -import org.apache.tika.io.IOUtils; +import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; -import org.eclipse.jetty.util.ajax.JSON; +import org.apache.tika.metadata.serialization.JsonMetadata; import javax.ws.rs.Produces; import javax.ws.rs.WebApplicationException; @@ -30,11 +30,10 @@ import java.io.IOException; import java.io.OutputStream; -import java.io.StringReader; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.lang.annotation.Annotation; import java.lang.reflect.Type; -import java.util.Map; -import java.util.TreeMap; @Provider @Produces(MediaType.APPLICATION_JSON) @@ -52,22 +51,13 @@ public void writeTo(Metadata metadata, Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException { - - Map res = new TreeMap(); - - for (String name : metadata.names()) { - String[] values = metadata.getValues(name); - if (metadata.isMultiValued(name)) { - res.put(name, values); - } else { - res.put(name, values[0]); - } - } - - String json = JSON.toString(res); - System.err.println("JSON : "+json); - StringReader r = new StringReader(json); - IOUtils.copy(r, entityStream); - entityStream.flush(); + try { + Writer writer = new OutputStreamWriter(entityStream, "UTF-8"); + JsonMetadata.toJson(metadata, writer); + writer.flush(); + } catch (TikaException e) { + throw new IOException(e); + } + entityStream.flush(); } } Index: tika-server/pom.xml =================================================================== --- tika-server/pom.xml (revision 1598698) +++ tika-server/pom.xml (working copy) @@ -34,6 +34,11 @@ ${project.version} + ${project.groupId} + tika-serialization + ${project.version} + + net.sf.opencsv opencsv 2.0 Index: tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java =================================================================== --- tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java (revision 0) +++ tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.tika.metadata.serialization; + +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import static org.junit.Assert.*; + +import java.io.StringReader; +import java.io.StringWriter; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.junit.Test; + +public class JsonMetadataTest { + + @Test + public void testBasicSerializationAndDeserialization() throws Exception { + Metadata metadata = new Metadata(); + metadata.add("k1", "v1"); + metadata.add("k1", "v2"); + metadata.add("k3", "v3"); + metadata.add("k4", "500,000"); + metadata.add("school", "\u666E\u6797\u65AF\u987F\u5927\u5B66"); + + StringWriter writer = new StringWriter(); + JsonMetadata.toJson(metadata, writer); + Metadata rebuilt = JsonMetadata.fromJson(new StringReader(writer.toString())); + assertEquals(metadata, rebuilt); + } + + + @Test + public void testDeserializationException() { + String json = "{\"k1\":[\"v1\",\"v2\"],\"k3\":\"v3\",\"k4\":500,000}"; + Metadata rebuilt; + boolean ex = false; + try { + rebuilt = JsonMetadata.fromJson(new StringReader(json)); + } catch (TikaException e) { + ex = true; + } + assertTrue(ex); + } + + + +} Index: tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java =================================================================== --- tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java (revision 0) +++ tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java (revision 0) @@ -0,0 +1,95 @@ +package org.apache.tika.metadata.serialization; + +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import java.lang.reflect.Type; +import java.util.Arrays; + +import org.apache.tika.metadata.Metadata; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + + +/** + * Serializer for metadata + * + */ +public class JsonMetadataSerializer implements JsonSerializer { + //Make sure to keep this static friendly for thread safety. + //Share state only with great caution. + + /** + * + * @param metadata + * @param type + * @param context + * @return JsonObject with key/value(s) pairs or JsonNull if metadata is null. + */ + @Override + public JsonElement serialize(Metadata metadata, Type type, JsonSerializationContext context) { + if (metadata == null){ + return new JsonNull(); + } + String[] names = getNames(metadata); + if (names == null) { + return new JsonNull(); + } + + JsonObject root = new JsonObject(); + + for (String n : names) { + + String[] vals = metadata.getValues(n); + if (vals == null) { + //silently skip? + continue; + } + + if (vals.length == 1){ + root.addProperty(n, vals[0]); + } else { + JsonArray jArr = new JsonArray(); + for (int i = 0; i < vals.length; i++) { + jArr.add(new JsonPrimitive(vals[i])); + } + root.add(n, jArr); + } + } + return root; + } + + /** + * Override to get a custom sort order + * or to filter names. + * + * @param metadata + * @return + */ + protected String[] getNames(Metadata metadata) { + String[] names = metadata.names(); + Arrays.sort(names); + return names; + } + +} Index: tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java =================================================================== --- tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java (revision 0) +++ tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java (revision 0) @@ -0,0 +1,91 @@ +package org.apache.tika.metadata.serialization; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.Reader; +import java.io.Writer; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonIOException; + +public class JsonMetadata { + + private static Gson GSON; + + static { + GsonBuilder builder = new GsonBuilder(); + builder.registerTypeHierarchyAdapter(Metadata.class, new JsonMetadataSerializer()); + builder.registerTypeHierarchyAdapter(Metadata.class, new JsonMetadataDeserializer()); + GSON = builder.create(); + } + + /** + * Serialize Metadata to writer + * @param metadata + * @param writer + */ + + /** + * Serializes a Metadata object to Json. This does not flush or close the writer. + * + * @param metadata + * @param writer + * @throws TikaException if there is an IOException during writing + */ + public static void toJson(Metadata metadata, Writer writer) throws TikaException { + try { + GSON.toJson(metadata, writer); + } catch (JsonIOException e) { + throw new TikaException(e.getMessage()); + } + } + + /** + * Read metadata from reader. + * + * @param reader + * @return Metadata or null if nothing could be read from the reader + * @throws TikaException in case of parse failure by Gson or IO failure with Reader + */ + public static Metadata fromJson(Reader reader) throws TikaException { + Metadata m = null; + try { + m = GSON.fromJson(reader, Metadata.class); + } catch (com.google.gson.JsonParseException e){ + //covers both io and parse exceptions + throw new TikaException(e.getMessage()); + } + return m; + } + + /** + * Enables setting custom configurations on Gson. Remember to register + * a serializer and a deserializer for Metadata. This does a literal set + * and does not add the default serializer and deserializers. + * + * @param gson + */ + public static void setGson(Gson gson) { + GSON = gson; + } +} Index: tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java =================================================================== --- tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java (revision 0) +++ tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java (revision 0) @@ -0,0 +1,68 @@ +package org.apache.tika.metadata.serialization; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.lang.reflect.Type; +import java.util.Iterator; +import java.util.Map; + +import org.apache.tika.metadata.Metadata; + +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; + + +/** + * Deserializer for metadata + * + */ +public class JsonMetadataDeserializer implements JsonDeserializer { + + //Make sure to keep this static friendly for thread safety. + //Share state only with great caution. + + + @Override + public Metadata deserialize(JsonElement element, Type type, + JsonDeserializationContext context) throws JsonParseException { + + final JsonObject obj = element.getAsJsonObject(); + Metadata m = new Metadata(); + for (Map.Entry entry : obj.entrySet()){ + String key = entry.getKey(); + JsonElement v = entry.getValue(); + if (v.isJsonPrimitive()){ + m.set(key, v.getAsString()); + } else if (v.isJsonArray()){ + JsonArray vArr = v.getAsJsonArray(); + Iterator itr = vArr.iterator(); + while (itr.hasNext()){ + JsonElement valueItem = itr.next(); + m.add(key, valueItem.getAsString()); + } + + } + } + return m; + } + +} Index: tika-serialization/pom.xml =================================================================== --- tika-serialization/pom.xml (revision 0) +++ tika-serialization/pom.xml (revision 0) @@ -0,0 +1,92 @@ + + + + + + 4.0.0 + + org.apache.tika + tika-parent + 1.6-SNAPSHOT + ../tika-parent/pom.xml + + + tika-serialization + + Apache Tika serialization + http://tika.apache.org + + + + ${project.groupId} + tika-core + ${project.version} + + + com.google.code.gson + gson + 1.7.1 + + + + + junit + junit + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + + The Apache Software Foundation + http://www.apache.org + + + http://svn.apache.org/viewvc/tika/trunk/tika-app + scm:svn:http://svn.apache.org/repos/asf/tika/trunk/tika-serialization + scm:svn:https://svn.apache.org/repos/asf/tika/trunk/tika-serialization + + + JIRA + https://issues.apache.org/jira/browse/TIKA + + + Jenkins + https://builds.apache.org/job/Tika-trunk/ + + + Index: tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java =================================================================== --- tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java (revision 1598698) +++ tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java (working copy) @@ -45,9 +45,6 @@ import javax.xml.transform.sax.TransformerHandler; import javax.xml.transform.stream.StreamResult; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.log4j.BasicConfigurator; @@ -75,6 +72,7 @@ import org.apache.tika.language.LanguageProfilerBuilder; import org.apache.tika.language.ProfilingHandler; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.serialization.JsonMetadata; import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MediaTypeRegistry; import org.apache.tika.mime.MimeTypeException; @@ -900,29 +898,25 @@ } } - /** - * Uses GSON. - */ private class NoDocumentJSONMetHandler extends DefaultHandler { - private final Gson gson; - protected final Metadata metadata; protected PrintWriter writer; - public NoDocumentJSONMetHandler(Metadata metadata, PrintWriter writer){ + public NoDocumentJSONMetHandler(Metadata metadata, PrintWriter writer) { this.metadata = metadata; this.writer = writer; - GsonBuilder builder = new GsonBuilder(); - builder.registerTypeHierarchyAdapter(Metadata.class, new JsonMetadataSerializer()); - gson = builder.create(); } @Override public void endDocument() throws SAXException { - gson.toJson(metadata, writer); + try { + JsonMetadata.toJson(metadata, writer); writer.flush(); - } - } + } catch (TikaException e) { + throw new SAXException(e); + } + } + } } Index: tika-app/pom.xml =================================================================== --- tika-app/pom.xml (revision 1598698) +++ tika-app/pom.xml (working copy) @@ -45,6 +45,11 @@ ${project.groupId} + tika-serialization + ${project.version} + + + ${project.groupId} tika-xmp ${project.version}