Index: /Users/joa23/Documents/workspace/nutch-trunk/src/java/org/apache/nutch/protocol/Content.java =================================================================== --- /Users/joa23/Documents/workspace/nutch-trunk/src/java/org/apache/nutch/protocol/Content.java (revision 357038) +++ /Users/joa23/Documents/workspace/nutch-trunk/src/java/org/apache/nutch/protocol/Content.java (working copy) @@ -76,11 +76,8 @@ contentType = UTF8.readString(in); // read contentType - int propertyCount = in.readInt(); // read metadata metadata = new ContentProperties(); - for (int i = 0; i < propertyCount; i++) { - metadata.put(UTF8.readString(in), UTF8.readString(in)); - } + metadata.readFields(in); // read meta data } public final void write(DataOutput out) throws IOException { @@ -93,13 +90,7 @@ UTF8.writeString(out, contentType); // write contentType - out.writeInt(metadata.size()); // write metadata - Iterator i = metadata.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry)i.next(); - UTF8.writeString(out, (String)e.getKey()); - UTF8.writeString(out, (String)e.getValue()); - } + metadata.write(out); // write metadata } public static Content read(DataInput in) throws IOException { Index: /Users/joa23/Documents/workspace/nutch-trunk/src/java/org/apache/nutch/protocol/ContentProperties.java =================================================================== --- /Users/joa23/Documents/workspace/nutch-trunk/src/java/org/apache/nutch/protocol/ContentProperties.java (revision 357038) +++ /Users/joa23/Documents/workspace/nutch-trunk/src/java/org/apache/nutch/protocol/ContentProperties.java (working copy) @@ -16,6 +16,10 @@ package org.apache.nutch.protocol; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; import java.util.Enumeration; import java.util.Iterator; import java.util.Properties; @@ -21,10 +25,13 @@ import java.util.Properties; import java.util.TreeMap; +import org.apache.nutch.io.UTF8; +import org.apache.nutch.io.Writable; + /** - * case insensitive properties + * writable case insensitive properties */ -public class ContentProperties extends TreeMap { +public class ContentProperties extends TreeMap implements Writable { /** * construct the TreeMap with a case insensitive comparator @@ -51,6 +58,36 @@ return (String) get(key); } + /* + * (non-Javadoc) + * + * @see java.util.Map#get(java.lang.Object) + */ + public Object get(Object arg0) { + Object object = super.get(arg0); + if (object != null && object instanceof ArrayList) { + ArrayList list = (ArrayList) object; + return list.get(list.size() - 1); + } + return object; + } + + /** + * @param key + * @return the properties as a string array if there is no such property we + * retunr a array with 0 entries + */ + public String[] getProperties(String key) { + Object object = super.get(key); + if (object != null && !(object instanceof ArrayList)) { + return new String[] { (String) object }; + } else if (object != null && object instanceof ArrayList) { + ArrayList list = (ArrayList) object; + return (String[]) list.toArray(new String[list.size()]); + } + return new String[0]; + } + /** * sets the key value tuple * @@ -58,7 +95,17 @@ * @param value */ public void setProperty(String key, String value) { - put(key, value); + Object object = super.get(key); + if (object != null && !(object instanceof ArrayList)) { + ArrayList arrayList = new ArrayList(); + arrayList.add(object); + arrayList.add(value); + put(key, arrayList); + } else if (object instanceof ArrayList) { + ((ArrayList) object).add(value); + } else { + put(key, value); + } } @@ -85,4 +132,72 @@ } + /* + * (non-Javadoc) + * + * @see org.apache.nutch.io.Writable#write(java.io.DataOutput) + */ + public final void write(DataOutput out) throws IOException { + out.writeInt(keySet().size()); + Iterator iterator = keySet().iterator(); + String key; + String[] properties; + while (iterator.hasNext()) { + key = (String) iterator.next(); + UTF8.writeString(out, key); + properties = getProperties(key); + out.writeInt(properties.length); + for (int i = 0; i < properties.length; i++) { + UTF8.writeString(out, properties[i]); + } + } + + } + + /* + * (non-Javadoc) + * + * @see org.apache.nutch.io.Writable#readFields(java.io.DataInput) + */ + public final void readFields(DataInput in) throws IOException { + int keySize = in.readInt(); + String key; + for (int i = 0; i < keySize; i++) { + key = UTF8.readString(in); + int valueSize = in.readInt(); + for (int j = 0; j < valueSize; j++) { + setProperty(key, UTF8.readString(in)); + } + } + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean equals(Object obj) { + if (!(obj instanceof ContentProperties)) { + return false; + } + ContentProperties properties = (ContentProperties) obj; + Enumeration enumeration = properties.propertyNames(); + while (enumeration.hasMoreElements()) { + String key = (String) enumeration.nextElement(); + String[] values = properties.getProperties(key); + String[] myValues = getProperties(key); + if (values.length != myValues.length) { + return false; + } + for (int i = 0; i < values.length; i++) { + if (!values[i].equals(myValues[i])) { + return false; + } + + } + } + + return true; + } + } Index: /Users/joa23/Documents/workspace/nutch-trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java =================================================================== --- /Users/joa23/Documents/workspace/nutch-trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java (revision 0) +++ /Users/joa23/Documents/workspace/nutch-trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java (revision 0) @@ -0,0 +1,64 @@ +/** + * Copyright 2005 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nutch.protocol; + +import org.apache.nutch.io.TestWritable; + +import junit.framework.TestCase; + +public class TestContentProperties extends TestCase { + + public void testOneValue() throws Exception { + ContentProperties properties = new ContentProperties(); + String value = "aValue"; + properties.setProperty("aKey", value); + assertEquals(value, properties.get("aKey")); + assertEquals(value, properties.get("akey")); + } + + public void testMultiValue() throws Exception { + ContentProperties properties = new ContentProperties(); + String value = "aValue"; + for (int i = 0; i < 100; i++) { + properties.setProperty("aKey", value + i); + + } + assertEquals(value + 99, properties.get("aKey")); + assertEquals(value + 99, properties.getProperty("aKey")); + String[] propertie = properties.getProperties("aKey"); + for (int i = 0; i < 100; i++) { + assertEquals(value + i, propertie[i]); + + } + } + + public void testSerialization() throws Exception { + ContentProperties properties = new ContentProperties(); + for (int i = 0; i < 10; i++) { + properties.setProperty("key", "" + i); + } + TestWritable.testWritable(properties); + Content content = new Content("url", "url", new byte[0], "text/html", + new ContentProperties()); + ContentProperties metadata = content.getMetadata(); + for (int i = 0; i < 100; i++) { + metadata.setProperty("aKey", "" + i); + } + TestWritable.testWritable(content); + } + +}