Index: src/test/org/apache/nutch/io/DummyWritable.java
===================================================================
--- src/test/org/apache/nutch/io/DummyWritable.java (revision 0)
+++ src/test/org/apache/nutch/io/DummyWritable.java (revision 0)
@@ -0,0 +1,31 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.io;
+
+import org.apache.nutch.io.IntWritable;
+
+public class DummyWritable extends IntWritable {
+
+ public DummyWritable() {
+
+ }
+
+ public DummyWritable(int i) {
+ super(i);
+ }
+
+}
Index: src/test/org/apache/nutch/io/TestMapWritable.java
===================================================================
--- src/test/org/apache/nutch/io/TestMapWritable.java (revision 0)
+++ src/test/org/apache/nutch/io/TestMapWritable.java (revision 0)
@@ -0,0 +1,158 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.io;
+
+import java.io.File;
+
+import junit.framework.TestCase;
+
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.fs.NutchFileSystem;
+import org.apache.nutch.io.SequenceFile.Reader;
+import org.apache.nutch.io.SequenceFile.Writer;
+
+public class TestMapWritable extends TestCase {
+
+ public void testMap() throws Exception {
+ MapWritable map = new MapWritable();
+ assertTrue(map.isEmpty());
+ for (int i = 0; i < 100; i++) {
+ UTF8 key = new UTF8("" + i);
+ IntWritable value = new IntWritable(i);
+ map.put(key, value);
+ assertEquals(i + 1, map.size());
+ assertTrue(map.containsKey(new UTF8("" + i)));
+ assertTrue(map.containsValue(new IntWritable(i)));
+ map.remove(key);
+ assertEquals(i, map.size());
+ map.put(key, value);
+ assertEquals(value, map.get(key));
+ assertFalse(map.isEmpty());
+ assertTrue(map.keySet().contains(key));
+ assertEquals(i + 1, map.values().size());
+ assertTrue(map.values().contains(value));
+ }
+ TestWritable.testWritable(map);
+ MapWritable map2 = new MapWritable();
+ TestWritable.testWritable(map2);
+ map2.putAll(map);
+ assertEquals(100, map2.size());
+ TestWritable.testWritable(map2);
+
+ map.clear();
+ assertTrue(map.isEmpty());
+ assertEquals(0, map.size());
+ assertFalse(map.containsKey(new UTF8("" + 1)));
+
+ }
+
+ public void testWritable() throws Exception {
+ MapWritable datum1 = new MapWritable();
+ for (int i = 0; i < 100; i++) {
+ datum1.put(new LongWritable(i), new UTF8("" + 1));
+ }
+ assertEquals(100, datum1.size());
+ TestWritable.testWritable(datum1);
+
+ MapWritable datum2 = new MapWritable();
+ for (int i = 0; i < 100; i++) {
+ datum2.put(new DummyWritable(i), new DummyWritable(i));
+ }
+ assertEquals(100, datum2.size());
+ TestWritable.testWritable(datum2);
+
+ CrawlDatum c = new CrawlDatum(CrawlDatum.STATUS_DB_FETCHED, 1f);
+ c.setMetaData(new MapWritable());
+ for (int i = 0; i < 100; i++) {
+ c.getMetaData().put(new LongWritable(i), new UTF8("" + 1));
+ }
+ TestWritable.testWritable(c);
+ }
+
+ public void testPerformance() throws Exception {
+ File file = new File(System.getProperty("java.io.tmpdir"),
+ "mapTestFile");
+ file.delete();
+ Writer writer = new SequenceFile.Writer(NutchFileSystem.get(), file
+ .getAbsolutePath(), IntWritable.class, MapWritable.class);
+ // write map
+ System.out.println("start writing");
+ long start = System.currentTimeMillis();
+ IntWritable key = new IntWritable();
+ MapWritable map = new MapWritable();
+ LongWritable mapValue = new LongWritable();
+ for (int i = 0; i < 1000000 ; i++) {
+ key.set(i);
+ mapValue.set(i);
+ map.put(key, mapValue);
+ writer.append(key, map);
+ }
+ long needed = System.currentTimeMillis() - start;
+ writer.close();
+ System.out.println("needed time for writing map: " + needed);
+
+// read map
+
+ Reader reader = new SequenceFile.Reader(NutchFileSystem.get(), file
+ .getAbsolutePath());
+ System.out.println("start reading");
+ start = System.currentTimeMillis();
+ while (reader.next(key, map)) {
+
+ }
+ reader.close();
+ needed = System.currentTimeMillis() - start;
+ System.out.println("needed time for reading map: " + needed);
+ file.delete();
+
+ // UTF8
+
+ writer = new SequenceFile.Writer(NutchFileSystem.get(), file
+ .getAbsolutePath(), IntWritable.class, UTF8.class);
+ // write map
+ start = System.currentTimeMillis();
+ key = new IntWritable();
+ UTF8 value = new UTF8();
+ String s = "15726:15726";
+ for (int i = 0; i < 1000000 ; i++) {
+ key.set(i);
+ value.set(s);
+ writer.append(key, value);
+ }
+ needed = System.currentTimeMillis() - start;
+ writer.close();
+ System.out.println("needed time for writing utf8: " + needed);
+
+ // read map
+
+ reader = new SequenceFile.Reader(NutchFileSystem.get(), file
+ .getAbsolutePath());
+ start = System.currentTimeMillis();
+ while (reader.next(key, value)) {
+
+ }
+ needed = System.currentTimeMillis() - start;
+ System.out.println("needed time for reading utf8: " + needed);
+ file.delete();
+
+
+ }
+
+ public static void main(String[] args) throws Exception {
+ TestMapWritable writable = new TestMapWritable();
+ writable.testPerformance();
+ }
+}
Index: src/java/org/apache/nutch/io/MapWritable.java
===================================================================
--- src/java/org/apache/nutch/io/MapWritable.java (revision 0)
+++ src/java/org/apache/nutch/io/MapWritable.java (revision 0)
@@ -0,0 +1,437 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import org.apache.nutch.util.LogFormatter;
+
+/**
+ * A writable map, having a similar behavior as java.util.HashMap
.
+ * MapWritable uses WritableName
to map key or values classes to
+ * a 1-byte id, that is stored additional to key and value data. Custom Writable
+ * classes that are not mapped in WritableName
are mapped to a id
+ * in a internal map. This internal map is also written into DataOutput. Maximum
+ * usable custom classes can be calculated by: 255-WritableName.getLastKey()
;
+ *
+ *
+ * @author Stefan Groschupf
+ */
+public class MapWritable implements Writable {
+
+ public static final Logger LOG = LogFormatter.getLogger(MapWritable.class
+ .getName());
+
+ private KeyValueEntry fFirst;
+
+ private KeyValueEntry fLast;
+
+ private KeyValueEntry fOld;
+
+ private int fSize = 0;
+
+ private int fIdCount = 0;
+
+ private ClassIdEntry fIdLast;
+
+ private ClassIdEntry fIdFirst;
+
+ public void clear() {
+ fOld = fFirst;
+ fFirst = fLast = null;
+ fSize = 0;
+ }
+
+ public boolean containsKey(Writable key) {
+ return findEntryByKey(key) != null;
+ }
+
+ public boolean containsValue(Writable value) {
+ KeyValueEntry entry = fFirst;
+ while (entry != null) {
+ if (entry.fValue.equals(value)) {
+ return true;
+ }
+ entry = entry.fNextEntry;
+ }
+ return false;
+ }
+
+ public Writable get(Writable key) {
+ KeyValueEntry entry = findEntryByKey(key);
+ if (entry != null) {
+ return entry.fValue;
+ }
+ return null;
+ }
+
+ public int hashCode() {
+ final int seed = 23;
+ int hash = 0;
+ KeyValueEntry entry = fFirst;
+ while (entry != null) {
+ hash += entry.fKey.hashCode() * seed;
+ hash += entry.fValue.hashCode() * seed;
+ entry = entry.fNextEntry;
+ }
+ return hash;
+
+ }
+
+ public boolean isEmpty() {
+ return fFirst == null;
+ }
+
+ public Set keySet() {
+ HashSet set = new HashSet();
+ set.add(fFirst.fKey);
+ KeyValueEntry entry = fFirst;
+ while ((entry = entry.fNextEntry) != null) {
+ set.add(entry.fKey);
+ }
+ return set;
+ }
+
+ public Writable put(Writable key, Writable value) {
+ KeyValueEntry entry = findEntryByKey(key);
+ if (entry != null) {
+ Writable oldValue = entry.fValue;
+ entry.fValue = value;
+ return oldValue;
+ }
+ KeyValueEntry newEntry = new KeyValueEntry(key, value);
+ fSize++;
+ if (fLast != null) {
+ fLast = fLast.fNextEntry = newEntry;
+ return null;
+ }
+ fLast = fFirst = newEntry;
+ return null;
+
+ }
+
+ public void putAll(MapWritable map) {
+ if (map == null || map.size() == 0) {
+ return;
+ }
+ Iterator iterator = map.keySet().iterator();
+ while (iterator.hasNext()) {
+ Writable key = (Writable) iterator.next();
+ Writable value = map.get(key);
+ put(key, value);
+ }
+ }
+
+ public Writable remove(Writable key) {
+ Writable oldValue = null;
+ KeyValueEntry entry = fFirst;
+ KeyValueEntry predecessor = null;
+ while (entry != null) {
+ if (entry.fKey.equals(key)) {
+ oldValue = entry.fValue;
+ if (predecessor == null) {
+ fFirst = fFirst.fNextEntry;
+ } else {
+ predecessor.fNextEntry = entry.fNextEntry;
+ }
+ if (fLast.equals(entry)) {
+ fLast = predecessor;
+ }
+ fSize--;
+ return oldValue;
+ }
+ predecessor = entry;
+ entry = entry.fNextEntry;
+ }
+ return oldValue;
+ }
+
+ public int size() {
+ return fSize;
+ }
+
+ public Collection values() {
+ LinkedList list = new LinkedList();
+ KeyValueEntry entry = fFirst;
+ while (entry != null) {
+ list.add(entry.fValue);
+ entry = entry.fNextEntry;
+ }
+ return list;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof MapWritable) {
+ MapWritable map = (MapWritable) obj;
+ KeyValueEntry e1 = fFirst;
+ KeyValueEntry e2 = map.fFirst;
+ while (e1 != null && e2 != null) {
+ if (!e1.equals(e2)) {
+ return false;
+ }
+ e1 = e1.fNextEntry;
+ e2 = e2.fNextEntry;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public String toString() {
+ if (fFirst != null) {
+ StringBuffer buffer = new StringBuffer();
+ KeyValueEntry entry = fFirst;
+ while (entry != null) {
+ buffer.append(entry.toString());
+ buffer.append(" ");
+ entry = entry.fNextEntry;
+ }
+ return buffer.toString();
+ }
+ return null;
+ }
+
+ private KeyValueEntry findEntryByKey(final Writable key) {
+ KeyValueEntry entry = fFirst;
+ while (entry != null && !entry.fKey.equals(key)) {
+ entry = entry.fNextEntry;
+ }
+ return entry;
+ }
+
+ // serialization methods
+
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(size());
+
+ if (size() > 0) {
+ // scan for unknown classes;
+ createInternalIdClassEntries();
+ // write internal map
+ out.writeByte(fIdCount);
+ if (fIdCount > 0) {
+ ClassIdEntry entry = fIdFirst;
+ while (entry != null) {
+ out.writeByte(entry.fId);
+ UTF8.writeString(out, entry.fclazz.getName());
+ entry = entry.fNextIdEntry;
+ }
+ }
+ // write meta data
+ KeyValueEntry entry = fFirst;
+ while (entry != null) {
+ out.writeByte(entry.fKeyClassId);
+ out.writeByte(entry.fValueClassId);
+
+ entry.fKey.write(out);
+ entry.fValue.write(out);
+
+ entry = entry.fNextEntry;
+ }
+
+ }
+
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ clear();
+ fSize = in.readInt();
+ if (fSize > 0) {
+ // read class-id map
+ fIdCount = in.readByte();
+ byte id;
+ Class clazz;
+ for (int i = 0; i < fIdCount; i++) {
+ try {
+ id = in.readByte();
+ clazz = Class.forName(UTF8.readString(in));
+ addIdEntry(id, clazz);
+ } catch (Exception e) {
+ LOG
+ .info("unable to load internal map entry"
+ + e.toString());
+ fIdCount--;
+ }
+ }
+ KeyValueEntry entry;
+ for (int i = 0; i < fSize; i++) {
+ try {
+ entry = getKeyValueEntry(in.readByte(), in.readByte());
+ entry.fKey.readFields(in);
+ entry.fValue.readFields(in);
+ if (fFirst == null) {
+ fFirst = fLast = entry;
+ } else {
+ fLast = fLast.fNextEntry = entry;
+ }
+ } catch (IOException e) {
+ LOG.warning("unable to load meta data entry, ignoring.. : "
+ + e.toString());
+ fSize--;
+ }
+ }
+ }
+ }
+
+ private void createInternalIdClassEntries() {
+ KeyValueEntry entry = fFirst;
+ byte id;
+ while (entry != null) {
+ id = getClassId(entry.fKey.getClass());
+ if (id == -128) {
+ id = addIdEntry(
+ (byte) (WritableName.getLastKey() + ++fIdCount),
+ entry.fKey.getClass());
+ }
+ entry.fKeyClassId = id;
+ id = getClassId(entry.fValue.getClass());
+ if (id == -128) {
+ id = addIdEntry(
+ (byte) (WritableName.getLastKey() + ++fIdCount),
+ entry.fValue.getClass());
+ }
+ entry.fValueClassId = id;
+ entry = entry.fNextEntry;
+ }
+ }
+
+ private byte addIdEntry(byte id, Class clazz) {
+ if (fIdFirst == null) {
+ fIdFirst = fIdLast = new ClassIdEntry(id, clazz);
+ } else {
+ fIdLast.fNextIdEntry = fIdLast = new ClassIdEntry(id, clazz);
+ }
+ return id;
+ }
+
+ private byte getClassId(Class clazz) {
+ byte classId = WritableName.getClassId(clazz);
+ if (classId != -128) {
+ return classId;
+ }
+ ClassIdEntry entry = fIdFirst;
+ while (entry != null) {
+ if (entry.fclazz.equals(clazz)) {
+ return entry.fId;
+ }
+ entry = entry.fNextIdEntry;
+ }
+ return -128;
+ }
+
+ private KeyValueEntry getKeyValueEntry(final byte keyId, final byte valueId)
+ throws IOException {
+ KeyValueEntry entry = fOld;
+ KeyValueEntry last = null;
+ byte entryKeyId;
+ byte entryValueId;
+ while (entry != null) {
+ entryKeyId = getClassId(entry.fKey.getClass());
+ entryValueId = getClassId(entry.fValue.getClass());
+ if (entryKeyId == keyId && entryValueId == valueId) {
+ if (last != null) {
+ last.fNextEntry = entry.fNextEntry;
+ } else {
+ fOld = entry.fNextEntry;
+ }
+ return entry;
+ }
+ last = entry;
+ entry = entry.fNextEntry;
+ }
+ Class keyClass = getClass(keyId);
+ Class valueClass = getClass(valueId);
+ try {
+ return new KeyValueEntry((Writable) keyClass.newInstance(),
+ (Writable) valueClass.newInstance());
+ } catch (Exception e) {
+ throw new IOException("unable to instantiate class: "
+ + e.toString());
+ }
+
+ }
+
+ private Class getClass(byte id) throws IOException {
+ Class clazz = WritableName.getClass(id);
+ if (clazz == null) {
+ ClassIdEntry entry = fIdFirst;
+ while (entry != null) {
+ if (entry.fId == id) {
+ return entry.fclazz;
+ }
+
+ entry = entry.fNextIdEntry;
+ }
+ } else {
+ return clazz;
+ }
+ throw new IOException("unable to load class for id: " + id);
+ }
+
+ /** an entry holds writable key and value */
+ private class KeyValueEntry {
+ private byte fKeyClassId;
+
+ private byte fValueClassId;
+
+ private Writable fKey;
+
+ private Writable fValue;
+
+ private KeyValueEntry fNextEntry;
+
+ public KeyValueEntry(Writable key, Writable value) {
+ this.fKey = key;
+ this.fValue = value;
+ }
+
+ public String toString() {
+ return fKey.toString() + ":" + fValue.toString();
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof KeyValueEntry) {
+ KeyValueEntry entry = (KeyValueEntry) obj;
+ return entry.fKey.equals(fKey) && entry.fValue.equals(fValue);
+ }
+ return false;
+ }
+ }
+
+ /** container for Id class tuples */
+ private class ClassIdEntry {
+ public ClassIdEntry(byte id, Class clazz) {
+ fId = id;
+ fclazz = clazz;
+ }
+
+ private byte fId;
+
+ private Class fclazz;
+
+ private ClassIdEntry fNextIdEntry;
+ }
+
+}
Index: src/java/org/apache/nutch/io/WritableName.java
===================================================================
--- src/java/org/apache/nutch/io/WritableName.java (revision 368453)
+++ src/java/org/apache/nutch/io/WritableName.java (working copy)
@@ -26,17 +26,39 @@
public class WritableName {
private static HashMap NAME_TO_CLASS = new HashMap();
private static HashMap CLASS_TO_NAME = new HashMap();
+ private static HashMap CLASS_TO_ID = new HashMap();
+ private static HashMap ID_TO_CLASS = new HashMap();
+ private static byte lastKey = (byte) -127;
static { // define important types
WritableName.setName(NullWritable.class, "null");
+ WritableName.setID(NullWritable.class, (byte)-126);
WritableName.setName(LongWritable.class, "long");
+ WritableName.setID(LongWritable.class, (byte)-125);
WritableName.setName(UTF8.class, "UTF8");
+ WritableName.setID(UTF8.class, (byte)-124);
WritableName.setName(MD5Hash.class, "MD5Hash");
+ WritableName.setID(MD5Hash.class, (byte)-123);
WritableName.setName
(org.apache.nutch.fetcher.FetcherOutput.class, "FetcherOutput");
+ WritableName.setID
+ (org.apache.nutch.fetcher.FetcherOutput.class, (byte)-122);
WritableName.setName(org.apache.nutch.protocol.Content.class, "Content");
+ WritableName.setID(org.apache.nutch.protocol.Content.class, (byte)-121);
WritableName.setName(org.apache.nutch.parse.ParseText.class, "ParseText");
+ WritableName.setID(org.apache.nutch.parse.ParseText.class, (byte)-120);
WritableName.setName(org.apache.nutch.parse.ParseData.class, "ParseData");
+ WritableName.setID(org.apache.nutch.parse.ParseData.class, (byte)-119);
+ WritableName.setName(MapWritable.class, "MapWritable");
+ WritableName.setID(MapWritable.class, (byte)-118);
+ WritableName.setName(BytesWritable.class, "BytesWritable");
+ WritableName.setID(BytesWritable.class, (byte)-117);
+ WritableName.setName(FloatWritable.class, "FloatWritable");
+ WritableName.setID(FloatWritable.class, (byte)-116);
+ WritableName.setName(IntWritable.class, "IntWritable");
+ WritableName.setID(IntWritable.class, (byte)-115);
+ WritableName.setName(ObjectWritable.class, "ObjectWritable");
+ WritableName.setID(ObjectWritable.class, (byte)-114);
}
private WritableName() {} // no public ctor
@@ -72,5 +94,40 @@
throw new IOException(e.toString());
}
}
+
+ /** Set the id for a writable class */
+ private static synchronized void setID(Class writableClass, byte id)
+ throws IllegalArgumentException {
+ if (ID_TO_CLASS.containsKey(new Byte(id))
+ || CLASS_TO_ID.containsKey(writableClass)) {
+ throw new IllegalArgumentException(
+ "id or class already in cache, id: " + id + " class: "
+ + writableClass.getName());
+ }
+ CLASS_TO_ID.put(writableClass, new Byte(id));
+ ID_TO_CLASS.put(new Byte(id), writableClass);
+ if (id > lastKey) {
+ lastKey = id;
+ }
+ }
+
+ /** Return a id for the given writable class */
+ public static byte getClassId(Class writableClass) {
+ Byte byteId = (Byte) CLASS_TO_ID.get(writableClass);
+ if (byteId != null) {
+ return byteId.byteValue();
+ }
+ return (byte)-128;
+ }
+
+ /** Return the class for a id. */
+ public static synchronized Class getClass(byte id) {
+ return (Class) ID_TO_CLASS.get(new Byte(id));
+ }
+
+ public static int getLastKey() {
+ return lastKey;
+ }
+
}
Index: src/java/org/apache/nutch/crawl/CrawlDatum.java
===================================================================
--- src/java/org/apache/nutch/crawl/CrawlDatum.java (revision 368453)
+++ src/java/org/apache/nutch/crawl/CrawlDatum.java (working copy)
@@ -17,12 +17,12 @@
package org.apache.nutch.crawl;
import java.io.*;
-import java.net.*;
import java.util.*;
import org.apache.nutch.io.*;
import org.apache.nutch.util.*;
+
/* The crawl state of a url. */
public class CrawlDatum implements WritableComparable, Cloneable {
public static final String DB_DIR_NAME = "current";
@@ -31,7 +31,7 @@
public static final String FETCH_DIR_NAME = "crawl_fetch";
public static final String PARSE_DIR_NAME = "crawl_parse";
- private final static byte CUR_VERSION = 3;
+ private final static byte CUR_VERSION = 4;
public static final byte STATUS_SIGNATURE = 0;
public static final byte STATUS_DB_UNFETCHED = 1;
@@ -62,6 +62,7 @@
private float score = 1.0f;
private byte[] signature = null;
private long modifiedTime;
+ private MapWritable metaDataMap;
public CrawlDatum() {}
@@ -117,7 +118,19 @@
throw new RuntimeException("Max signature length (256) exceeded: " + signature.length);
this.signature = signature;
}
+
+ public void setMetaData(MapWritable mapWritable) {this.metaDataMap = mapWritable; }
+ /**
+ * returns a MapWritable if it was setted or read @see readFields(DataInput),
+ * returns null in case CrawlDatum was freshly generated or a empty map
+ * in case CrawlDatum is a recyled instance.
+ */
+ public MapWritable getMetaData() {
+ return this.metaDataMap;
+ }
+
+
//
// writable methods
//
@@ -147,6 +160,20 @@
in.readFully(signature);
} else signature = null;
}
+ if (version > 3) {
+ if (in.readBoolean()) {
+ if(metaDataMap==null){
+ metaDataMap = new MapWritable();
+ } else {
+ metaDataMap.clear();
+ }
+ metaDataMap.readFields(in);
+ } else {
+ if(metaDataMap !=null){
+ metaDataMap.clear(); // at least clear old meta data
+ }
+ }
+ }
}
/** The number of bytes into a CrawlDatum that the score is stored. */
@@ -167,6 +194,12 @@
out.writeByte(signature.length);
out.write(signature);
}
+ if(metaDataMap!=null && metaDataMap.size()>0){
+ out.writeBoolean(true);
+ metaDataMap.write(out);
+ } else {
+ out.writeBoolean(false);
+ }
}
/** Copy the contents of another instance into this instance. */
@@ -178,6 +211,7 @@
this.score = that.score;
this.modifiedTime = that.modifiedTime;
this.signature = that.signature;
+ this.metaDataMap = that.metaDataMap;
}