Index: build.xml
===================================================================
--- build.xml (revision 1650154)
+++ build.xml (working copy)
@@ -559,9 +559,10 @@
-
+
-
+
+
Index: ivy/ivy.xml
===================================================================
--- ivy/ivy.xml (revision 1650154)
+++ ivy/ivy.xml (working copy)
@@ -135,6 +135,11 @@
+
+
+
Index: src/gora/host.avsc
===================================================================
--- src/gora/host.avsc (revision 1650154)
+++ src/gora/host.avsc (working copy)
@@ -1,9 +1,41 @@
-{"name": "Host",
- "type": "record",
- "namespace": "org.apache.nutch.storage",
- "fields": [
- {"name": "metadata", "type": {"type": "map", "values": "bytes"}, "default":{}},
- {"name": "outlinks", "type": {"type": "map", "values": "string"}, "default":{}},
- {"name": "inlinks", "type": {"type": "map", "values": "string"}, "default":{}}
- ]
+{
+ "name": "Host",
+ "type": "record",
+ "namespace": "org.apache.nutch.storage",
+ "doc": "Host represents a store of webpages or other data which resides on a server or other computer so that it can be accessed over the Internet",
+ "fields": [
+ {
+ "name": "metadata",
+ "type": {
+ "type": "map",
+ "values": ["null","bytes"]
+ },
+ "doc": "A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc",
+ "default": {
+
+ }
+ },
+ {
+ "name": "outlinks",
+ "type": {
+ "type": "map",
+ "values": ["null","string"]
+ },
+ "doc": "Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics",
+ "default": {
+
+ }
+ },
+ {
+ "name": "inlinks",
+ "type": {
+ "type": "map",
+ "values": ["null","string"]
+ },
+ "doc": "Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics",
+ "default": {
+
+ }
+ }
+ ]
}
Index: src/gora/webpage.avsc
===================================================================
--- src/gora/webpage.avsc (revision 1650154)
+++ src/gora/webpage.avsc (working copy)
@@ -1,48 +1,285 @@
-{"name": "WebPage",
- "type": "record",
- "namespace": "org.apache.nutch.storage",
- "fields": [
- {"name": "baseUrl", "type": ["null","string"], "default":null},
- {"name": "status", "type": "int", "default":0},
- {"name": "fetchTime", "type": "long", "default":0},
- {"name": "prevFetchTime", "type": "long", "default":0},
- {"name": "fetchInterval", "type": "int", "default":0},
- {"name": "retriesSinceFetch", "type": "int", "default":0},
- {"name": "modifiedTime", "type": "long", "default":0},
- {"name": "prevModifiedTime", "type": "long", "default":0},
- {"name": "protocolStatus", "type": ["null", {
- "name": "ProtocolStatus",
- "type": "record",
- "namespace": "org.apache.nutch.storage",
- "fields": [
- {"name": "code", "type": "int", "default":0},
- {"name": "args", "type": {"type": "array", "items": "string"}, "default":[]},
- {"name": "lastModified", "type": "long", "default":0}
- ]
- }], "default":null},
- {"name": "content", "type": ["null","bytes"], "default":null},
- {"name": "contentType", "type": ["null","string"], "default":null},
- {"name": "prevSignature", "type": ["null","bytes"], "default":null},
- {"name": "signature", "type": ["null","bytes"], "default":null},
- {"name": "title", "type": ["null","string"], "default":null},
- {"name": "text", "type": ["null","string"], "default":null},
- {"name": "parseStatus", "type": ["null", {
- "name": "ParseStatus",
- "type": "record",
- "namespace": "org.apache.nutch.storage",
- "fields": [
- {"name": "majorCode", "type": "int", "default":0},
- {"name": "minorCode", "type": "int", "default":0},
- {"name": "args", "type": {"type": "array", "items": "string"}, "default":[]}
- ]
- }], "default":null},
- {"name": "score", "type": "float", "default":0},
- {"name": "reprUrl", "type": ["null","string"], "default":null},
- {"name": "headers", "type": {"type":"map", "values": ["null","string"]}, "default":{}},
- {"name": "outlinks", "type": {"type": "map", "values": ["null","string"]}, "default":{}},
- {"name": "inlinks", "type": {"type": "map", "values": ["null","string"]}, "default":{}},
- {"name": "markers", "type": {"type": "map", "values": ["null","string"]}, "default":{}},
- {"name": "metadata", "type": {"type": "map", "values": ["null","bytes"]}, "default":{}},
- {"name": "batchId", "type": ["null","string"], "default":null}
- ]
+{
+ "name": "WebPage",
+ "type": "record",
+ "namespace": "org.apache.nutch.storage",
+ "doc": "WebPage is the primary data structure in Nutch representing crawl data for a given WebPage at some point in time",
+ "fields": [
+ {
+ "name": "baseUrl",
+ "type": [
+ "null",
+ "string"
+ ],
+ "doc": "The original associated with this WebPage.",
+ "default": null
+ },
+ {
+ "name": "status",
+ "type": "int",
+ "doc": "A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified",
+ "default": 0
+ },
+ {
+ "name": "fetchTime",
+ "type": "long",
+ "doc": "The system time in milliseconds for when the page was fetched.",
+ "default": 0
+ },
+ {
+ "name": "prevFetchTime",
+ "type": "long",
+ "doc": "The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation",
+ "default": 0
+ },
+ {
+ "name": "fetchInterval",
+ "type": "int",
+ "doc": "The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented.",
+ "default": 0
+ },
+ {
+ "name": "retriesSinceFetch",
+ "type": "int",
+ "doc": "The number of retried attempts at fetching the WebPage since it was last successfully fetched.",
+ "default": 0
+ },
+ {
+ "name": "modifiedTime",
+ "type": "long",
+ "doc": "The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage.",
+ "default": 0
+ },
+ {
+ "name": "prevModifiedTime",
+ "type": "long",
+ "doc": "The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage.",
+ "default": 0
+ },
+ {
+ "name": "protocolStatus",
+ "type": [
+ "null",
+ {
+ "name": "ProtocolStatus",
+ "type": "record",
+ "namespace": "org.apache.nutch.storage",
+ "doc": "A nested container representing data captured from web server responses.",
+ "fields": [
+ {
+ "name": "code",
+ "type": "int",
+ "doc": "A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing\/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching.",
+ "default": 0
+ },
+ {
+ "name": "args",
+ "type": {
+ "type": "array",
+ "items": "string"
+ },
+ "doc": "Optional arguments supplied to compliment and\/or justify the response code.",
+ "default": [
+
+ ]
+ },
+ {
+ "name": "lastModified",
+ "type": "long",
+ "doc": "A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself.",
+ "default": 0
+ }
+ ]
+ }
+ ],
+ "default": null
+ },
+ {
+ "name": "content",
+ "type": [
+ "null",
+ "bytes"
+ ],
+ "doc": "The entire raw document content e.g. raw XHTML",
+ "default": null
+ },
+ {
+ "name": "contentType",
+ "type": [
+ "null",
+ "string"
+ ],
+ "doc": "The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used.",
+ "default": null
+ },
+ {
+ "name": "prevSignature",
+ "type": [
+ "null",
+ "bytes"
+ ],
+ "doc": "An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints.",
+ "default": null
+ },
+ {
+ "name": "signature",
+ "type": [
+ "null",
+ "bytes"
+ ],
+ "doc": "An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time.",
+ "default": null
+ },
+ {
+ "name": "title",
+ "type": [
+ "null",
+ "string"
+ ],
+ "doc": "The title of the WebPage.",
+ "default": null
+ },
+ {
+ "name": "text",
+ "type": [
+ "null",
+ "string"
+ ],
+ "doc": "The textual content of the WebPage devoid from native markup.",
+ "default": null
+ },
+ {
+ "name": "parseStatus",
+ "type": [
+ "null",
+ {
+ "name": "ParseStatus",
+ "type": "record",
+ "namespace": "org.apache.nutch.storage",
+ "doc": "A nested container representing parse status data captured from invocation of parsers on fetch of a WebPage",
+ "fields": [
+ {
+ "name": "majorCode",
+ "type": "int",
+ "doc": "Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.)",
+ "default": 0
+ },
+ {
+ "name": "minorCode",
+ "type": "int",
+ "doc": "Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage.",
+ "default": 0
+ },
+ {
+ "name": "args",
+ "type": {
+ "type": "array",
+ "items": "string"
+ },
+ "doc": "Optional arguments supplied to compliment and\/or justify the parse status code.",
+ "default": [
+
+ ]
+ }
+ ]
+ }
+ ],
+ "default": null
+ },
+ {
+ "name": "score",
+ "type": "float",
+ "doc": "A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics.",
+ "default": 0
+ },
+ {
+ "name": "reprUrl",
+ "type": [
+ "null",
+ "string"
+ ],
+ "doc": "In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler",
+ "default": null
+ },
+ {
+ "name": "headers",
+ "type": {
+ "type": "map",
+ "values": [
+ "null",
+ "string"
+ ]
+ },
+ "doc": "Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION.",
+ "default": {
+
+ }
+ },
+ {
+ "name": "outlinks",
+ "type": {
+ "type": "map",
+ "values": [
+ "null",
+ "string"
+ ]
+ },
+ "doc": "Embedded hyperlinks which direct outside of the current domain.",
+ "default": {
+
+ }
+ },
+ {
+ "name": "inlinks",
+ "type": {
+ "type": "map",
+ "values": [
+ "null",
+ "string"
+ ]
+ },
+ "doc": "Embedded hyperlinks which link to pages within the current domain.",
+ "default": {
+
+ }
+ },
+ {
+ "name": "markers",
+ "type": {
+ "type": "map",
+ "values": [
+ "null",
+ "string"
+ ]
+ },
+ "doc": "Markers flags which represent user and machine decisions which have affected influenced a WebPage's current state. Markers can be system specific and user machine driven in nature. They are assigned to a WebPage on a job-by-job basis and thier values indicative of what actions should be associated with a WebPage.",
+ "default": {
+
+ }
+ },
+ {
+ "name": "metadata",
+ "type": {
+ "type": "map",
+ "values": [
+ "null",
+ "bytes"
+ ]
+ },
+ "doc": "A multi-valued metadata container used for storing everything from structured WebPage characterists, to ad-hoc extraction and metadata augmentation for any given WebPage.",
+ "default": {
+
+ }
+ },
+ {
+ "name": "batchId",
+ "type": [
+ "null",
+ "string"
+ ],
+ "doc": "A batchId that this WebPage is assigned to. WebPage's are fetched in batches, called fetchlists. Pages are partitioned but can always be associated and fetched alongside pages of similar value (within a crawl cycle) based on batchId.",
+ "default": null
+ }
+ ]
}
+
+
Index: src/java/org/apache/nutch/storage/Host.java
===================================================================
--- src/java/org/apache/nutch/storage/Host.java (revision 1650154)
+++ src/java/org/apache/nutch/storage/Host.java (working copy)
@@ -1,39 +1,39 @@
/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+******************************************************************************/
/**
* Autogenerated by Avro
*
* DO NOT EDIT DIRECTLY
*/
-package org.apache.nutch.storage;
+package org.apache.nutch.storage;
import org.apache.avro.util.Utf8;
import org.apache.nutch.util.Bytes;
@SuppressWarnings("all")
+/** Host represents a store of webpages or other data which resides on a server or other computer so that it can be accessed over the Internet */
public class Host extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Host\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"},\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Host\",\"namespace\":\"org.apache.nutch.storage\",\"doc\":\"Host represents a store of webpages or other data which resides on a server or other computer so that it can be accessed over the Internet\",\"fields\":[{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"bytes\"]},\"doc\":\"A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc\",\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"doc\":\"Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics\",\"default\":{}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"doc\":\"Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics\",\"default\":{}}]}");
/** Enum containing all data bean's fields. */
public static enum Field {
- __G__DIRTY(0, "__g__dirty"),
- METADATA(1, "metadata"),
- OUTLINKS(2, "outlinks"),
- INLINKS(3, "inlinks"),
+ METADATA(0, "metadata"),
+ OUTLINKS(1, "outlinks"),
+ INLINKS(2, "inlinks"),
;
/**
* Field's index.
@@ -72,12 +72,11 @@
};
public static final String[] _ALL_FIELDS = {
- "__g__dirty",
"metadata",
"outlinks",
"inlinks",
};
-
+
/**
* Gets the total field count.
* @return int field count
@@ -84,21 +83,21 @@
*/
public int getFieldsCount() {
return Host._ALL_FIELDS.length;
- }
+ }
- /** Bytes used to represent weather or not a field is dirty. */
- private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]);
- private java.util.Map metadata;
- private java.util.Map outlinks;
- private java.util.Map inlinks;
+ /** A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc */
+ private java.util.Map metadata;
+ /** Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics */
+ private java.util.Map outlinks;
+ /** Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics */
+ private java.util.Map inlinks;
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
- public Object get(int field$) {
+ public java.lang.Object get(int field$) {
switch (field$) {
- case 0: return __g__dirty;
- case 1: return metadata;
- case 2: return outlinks;
- case 3: return inlinks;
+ case 0: return metadata;
+ case 1: return outlinks;
+ case 2: return inlinks;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -105,12 +104,11 @@
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
- public void put(int field$, Object value) {
+ public void put(int field$, java.lang.Object value) {
switch (field$) {
- case 0: __g__dirty = (java.nio.ByteBuffer)(value); break;
- case 1: metadata = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 2: outlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 3: inlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 0: metadata = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 1: outlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 2: inlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -117,114 +115,92 @@
/**
* Gets the value of the 'metadata' field.
- */
- public java.util.Map getMetadata() {
+ * A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc */
+ public java.util.Map getMetadata() {
return metadata;
}
/**
* Sets the value of the 'metadata' field.
- * @param value the value to set.
+ * A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc * @param value the value to set.
*/
- public void setMetadata(java.util.Map value) {
+ public void setMetadata(java.util.Map value) {
this.metadata = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(1);
+ setDirty(0);
}
/**
* Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc * @param value the value to set.
*/
- public boolean isMetadataDirty(java.util.Map value) {
- return isDirty(1);
+ public boolean isMetadataDirty(java.util.Map value) {
+ return isDirty(0);
}
/**
* Gets the value of the 'outlinks' field.
- */
- public java.util.Map getOutlinks() {
+ * Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics */
+ public java.util.Map getOutlinks() {
return outlinks;
}
/**
* Sets the value of the 'outlinks' field.
- * @param value the value to set.
+ * Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public void setOutlinks(java.util.Map value) {
+ public void setOutlinks(java.util.Map value) {
this.outlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(2);
+ setDirty(1);
}
/**
* Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public boolean isOutlinksDirty(java.util.Map value) {
- return isDirty(2);
+ public boolean isOutlinksDirty(java.util.Map value) {
+ return isDirty(1);
}
/**
* Gets the value of the 'inlinks' field.
- */
- public java.util.Map getInlinks() {
+ * Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics */
+ public java.util.Map getInlinks() {
return inlinks;
}
/**
* Sets the value of the 'inlinks' field.
- * @param value the value to set.
+ * Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public void setInlinks(java.util.Map value) {
+ public void setInlinks(java.util.Map value) {
this.inlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(3);
+ setDirty(2);
}
/**
* Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public boolean isInlinksDirty(java.util.Map value) {
- return isDirty(3);
+ public boolean isInlinksDirty(java.util.Map value) {
+ return isDirty(2);
}
- public boolean contains(String key) {
- return metadata.containsKey(new Utf8(key));
- }
-
- public String getValue(String key, String defaultValue) {
- if (!contains(key))
- return defaultValue;
- return Bytes.toString(metadata.get(new Utf8(key)));
- }
-
- public int getInt(String key, int defaultValue) {
- if (!contains(key))
- return defaultValue;
- return Integer.parseInt(getValue(key, null));
- }
-
- public long getLong(String key, long defaultValue) {
- if (!contains(key))
- return defaultValue;
- return Long.parseLong(getValue(key, null));
- }
-
/** Creates a new Host RecordBuilder */
- public static Builder newBuilder() {
- return new Builder();
+ public static org.apache.nutch.storage.Host.Builder newBuilder() {
+ return new org.apache.nutch.storage.Host.Builder();
}
/** Creates a new Host RecordBuilder by copying an existing Builder */
- public static Builder newBuilder(Builder other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.Host.Builder newBuilder(org.apache.nutch.storage.Host.Builder other) {
+ return new org.apache.nutch.storage.Host.Builder(other);
}
/** Creates a new Host RecordBuilder by copying an existing Host instance */
- public static Builder newBuilder(Host other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.Host.Builder newBuilder(org.apache.nutch.storage.Host other) {
+ return new org.apache.nutch.storage.Host.Builder(other);
}
- private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer(
+ private static java.nio.ByteBuffer deepCopyToReadOnlyBuffer(
java.nio.ByteBuffer input) {
java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity());
int position = input.position();
@@ -253,114 +229,109 @@
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase
implements org.apache.avro.data.RecordBuilder {
- private java.nio.ByteBuffer __g__dirty;
- private java.util.Map metadata;
- private java.util.Map outlinks;
- private java.util.Map inlinks;
+ private java.util.Map metadata;
+ private java.util.Map outlinks;
+ private java.util.Map inlinks;
/** Creates a new Builder */
private Builder() {
- super(Host.SCHEMA$);
+ super(org.apache.nutch.storage.Host.SCHEMA$);
}
/** Creates a Builder by copying an existing Builder */
- private Builder(Builder other) {
+ private Builder(org.apache.nutch.storage.Host.Builder other) {
super(other);
}
/** Creates a Builder by copying an existing Host instance */
- private Builder(Host other) {
- super(Host.SCHEMA$);
- if (isValidValue(fields()[0], other.__g__dirty)) {
- this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty);
+ private Builder(org.apache.nutch.storage.Host other) {
+ super(org.apache.nutch.storage.Host.SCHEMA$);
+ if (isValidValue(fields()[0], other.metadata)) {
+ this.metadata = (java.util.Map) data().deepCopy(fields()[0].schema(), other.metadata);
fieldSetFlags()[0] = true;
}
- if (isValidValue(fields()[1], other.metadata)) {
- this.metadata = (java.util.Map) data().deepCopy(fields()[1].schema(), other.metadata);
+ if (isValidValue(fields()[1], other.outlinks)) {
+ this.outlinks = (java.util.Map) data().deepCopy(fields()[1].schema(), other.outlinks);
fieldSetFlags()[1] = true;
}
- if (isValidValue(fields()[2], other.outlinks)) {
- this.outlinks = (java.util.Map) data().deepCopy(fields()[2].schema(), other.outlinks);
+ if (isValidValue(fields()[2], other.inlinks)) {
+ this.inlinks = (java.util.Map) data().deepCopy(fields()[2].schema(), other.inlinks);
fieldSetFlags()[2] = true;
}
- if (isValidValue(fields()[3], other.inlinks)) {
- this.inlinks = (java.util.Map) data().deepCopy(fields()[3].schema(), other.inlinks);
- fieldSetFlags()[3] = true;
- }
}
/** Gets the value of the 'metadata' field */
- public java.util.Map getMetadata() {
+ public java.util.Map getMetadata() {
return metadata;
}
/** Sets the value of the 'metadata' field */
- public Builder setMetadata(java.util.Map value) {
- validate(fields()[1], value);
+ public org.apache.nutch.storage.Host.Builder setMetadata(java.util.Map value) {
+ validate(fields()[0], value);
this.metadata = value;
- fieldSetFlags()[1] = true;
+ fieldSetFlags()[0] = true;
return this;
}
/** Checks whether the 'metadata' field has been set */
public boolean hasMetadata() {
- return fieldSetFlags()[1];
+ return fieldSetFlags()[0];
}
/** Clears the value of the 'metadata' field */
- public Builder clearMetadata() {
+ public org.apache.nutch.storage.Host.Builder clearMetadata() {
metadata = null;
- fieldSetFlags()[1] = false;
+ fieldSetFlags()[0] = false;
return this;
}
/** Gets the value of the 'outlinks' field */
- public java.util.Map getOutlinks() {
+ public java.util.Map getOutlinks() {
return outlinks;
}
/** Sets the value of the 'outlinks' field */
- public Builder setOutlinks(java.util.Map value) {
- validate(fields()[2], value);
+ public org.apache.nutch.storage.Host.Builder setOutlinks(java.util.Map value) {
+ validate(fields()[1], value);
this.outlinks = value;
- fieldSetFlags()[2] = true;
+ fieldSetFlags()[1] = true;
return this;
}
/** Checks whether the 'outlinks' field has been set */
public boolean hasOutlinks() {
- return fieldSetFlags()[2];
+ return fieldSetFlags()[1];
}
/** Clears the value of the 'outlinks' field */
- public Builder clearOutlinks() {
+ public org.apache.nutch.storage.Host.Builder clearOutlinks() {
outlinks = null;
- fieldSetFlags()[2] = false;
+ fieldSetFlags()[1] = false;
return this;
}
/** Gets the value of the 'inlinks' field */
- public java.util.Map getInlinks() {
+ public java.util.Map getInlinks() {
return inlinks;
}
/** Sets the value of the 'inlinks' field */
- public Builder setInlinks(java.util.Map value) {
- validate(fields()[3], value);
+ public org.apache.nutch.storage.Host.Builder setInlinks(java.util.Map value) {
+ validate(fields()[2], value);
this.inlinks = value;
- fieldSetFlags()[3] = true;
+ fieldSetFlags()[2] = true;
return this;
}
/** Checks whether the 'inlinks' field has been set */
public boolean hasInlinks() {
- return fieldSetFlags()[3];
+ return fieldSetFlags()[2];
}
/** Clears the value of the 'inlinks' field */
- public Builder clearInlinks() {
+ public org.apache.nutch.storage.Host.Builder clearInlinks() {
inlinks = null;
- fieldSetFlags()[3] = false;
+ fieldSetFlags()[2] = false;
return this;
}
@@ -368,10 +339,9 @@
public Host build() {
try {
Host record = new Host();
- record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]);
- record.metadata = fieldSetFlags()[1] ? this.metadata : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[1]));
- record.outlinks = fieldSetFlags()[2] ? this.outlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[2]));
- record.inlinks = fieldSetFlags()[3] ? this.inlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[3]));
+ record.metadata = fieldSetFlags()[0] ? this.metadata : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[0]));
+ record.outlinks = fieldSetFlags()[1] ? this.outlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[1]));
+ record.inlinks = fieldSetFlags()[2] ? this.inlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[2]));
return record;
} catch (Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
@@ -379,7 +349,7 @@
}
}
- public Tombstone getTombstone(){
+ public Host.Tombstone getTombstone(){
return TOMBSTONE;
}
@@ -387,6 +357,29 @@
return newBuilder().build();
}
+ // TODO NUTCH-1709 Generated classes o.a.n.storage.Host and o.a.n.storage.ProtocolStatus contain methods not defined in source .avsc
+ public boolean contains(String key) {
+ return metadata.containsKey(new Utf8(key));
+ }
+
+ // TODO NUTCH-1709 Generated classes o.a.n.storage.Host and o.a.n.storage.ProtocolStatus contain methods not defined in source .avsc
+ public String getValue(String key, String defaultValue) {
+ if (!contains(key)) return defaultValue;
+ return Bytes.toString(metadata.get(new Utf8(key)));
+ }
+
+ // TODO NUTCH-1709 Generated classes o.a.n.storage.Host and o.a.n.storage.ProtocolStatus contain methods not defined in source .avsc
+ public int getInt(String key, int defaultValue) {
+ if (!contains(key)) return defaultValue;
+ return Integer.parseInt(getValue(key,null));
+ }
+
+ // TODO NUTCH-1709 Generated classes o.a.n.storage.Host and o.a.n.storage.ProtocolStatus contain methods not defined in source .avsc
+ public long getLong(String key, long defaultValue) {
+ if (!contains(key)) return defaultValue;
+ return Long.parseLong(getValue(key,null));
+ }
+
private static final Tombstone TOMBSTONE = new Tombstone();
public static final class Tombstone extends Host implements org.apache.gora.persistency.Tombstone {
@@ -393,76 +386,77 @@
private Tombstone() { }
- /**
+ /**
* Gets the value of the 'metadata' field.
- */
- public java.util.Map getMetadata() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc */
+ public java.util.Map getMetadata() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'metadata' field.
- * @param value the value to set.
+ * A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc * @param value the value to set.
*/
- public void setMetadata(java.util.Map value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setMetadata(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A multivalued metadata container used for storing a wide variety of host metadata such as structured web server characterists etc * @param value the value to set.
*/
- public boolean isMetadataDirty(java.util.Map value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isMetadataDirty(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'outlinks' field.
- */
- public java.util.Map getOutlinks() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics */
+ public java.util.Map getOutlinks() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'outlinks' field.
- * @param value the value to set.
+ * Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public void setOutlinks(java.util.Map value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setOutlinks(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Hyperlinks which direct outside of the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public boolean isOutlinksDirty(java.util.Map value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isOutlinksDirty(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'inlinks' field.
- */
- public java.util.Map getInlinks() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics */
+ public java.util.Map getInlinks() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'inlinks' field.
- * @param value the value to set.
+ * Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public void setInlinks(java.util.Map value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setInlinks(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Hyperlinks which link to pages within the current host domain these can used in a histogram style manner to generate host statistics * @param value the value to set.
*/
- public boolean isInlinksDirty(java.util.Map value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isInlinksDirty(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
}
-}
\ No newline at end of file
+}
+
Index: src/java/org/apache/nutch/storage/ParseStatus.java
===================================================================
--- src/java/org/apache/nutch/storage/ParseStatus.java (revision 1650154)
+++ src/java/org/apache/nutch/storage/ParseStatus.java (working copy)
@@ -1,19 +1,19 @@
/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+******************************************************************************/
/**
* Autogenerated by Avro
*
@@ -21,15 +21,15 @@
*/
package org.apache.nutch.storage;
@SuppressWarnings("all")
+/** A nested container representing parse status data captured from invocation of parsers on fetch of a WebPage */
public class ParseStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"majorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ParseStatus\",\"namespace\":\"org.apache.nutch.storage\",\"doc\":\"A nested container representing parse status data captured from invocation of parsers on fetch of a WebPage\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\",\"doc\":\"Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.)\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"doc\":\"Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage.\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"doc\":\"Optional arguments supplied to compliment and/or justify the parse status code.\",\"default\":[]}]}");
/** Enum containing all data bean's fields. */
public static enum Field {
- __G__DIRTY(0, "__g__dirty"),
- MAJOR_CODE(1, "majorCode"),
- MINOR_CODE(2, "minorCode"),
- ARGS(3, "args"),
+ MAJOR_CODE(0, "majorCode"),
+ MINOR_CODE(1, "minorCode"),
+ ARGS(2, "args"),
;
/**
* Field's index.
@@ -68,12 +68,11 @@
};
public static final String[] _ALL_FIELDS = {
- "__g__dirty",
"majorCode",
"minorCode",
"args",
};
-
+
/**
* Gets the total field count.
* @return int field count
@@ -82,19 +81,19 @@
return ParseStatus._ALL_FIELDS.length;
}
- /** Bytes used to represent weather or not a field is dirty. */
- private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]);
+ /** Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) */
private int majorCode;
+ /** Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. */
private int minorCode;
- private java.util.List args;
+ /** Optional arguments supplied to compliment and/or justify the parse status code. */
+ private java.util.List args;
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
- public Object get(int field$) {
+ public java.lang.Object get(int field$) {
switch (field$) {
- case 0: return __g__dirty;
- case 1: return majorCode;
- case 2: return minorCode;
- case 3: return args;
+ case 0: return majorCode;
+ case 1: return minorCode;
+ case 2: return args;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -101,12 +100,11 @@
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
- public void put(int field$, Object value) {
+ public void put(int field$, java.lang.Object value) {
switch (field$) {
- case 0: __g__dirty = (java.nio.ByteBuffer)(value); break;
- case 1: majorCode = (Integer)(value); break;
- case 2: minorCode = (Integer)(value); break;
- case 3: args = (java.util.List)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break;
+ case 0: majorCode = (java.lang.Integer)(value); break;
+ case 1: minorCode = (java.lang.Integer)(value); break;
+ case 2: args = (java.util.List)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -113,92 +111,92 @@
/**
* Gets the value of the 'majorCode' field.
- */
- public Integer getMajorCode() {
+ * Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) */
+ public java.lang.Integer getMajorCode() {
return majorCode;
}
/**
* Sets the value of the 'majorCode' field.
- * @param value the value to set.
+ * Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) * @param value the value to set.
*/
- public void setMajorCode(Integer value) {
+ public void setMajorCode(java.lang.Integer value) {
this.majorCode = value;
- setDirty(1);
+ setDirty(0);
}
/**
* Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) * @param value the value to set.
*/
- public boolean isMajorCodeDirty(Integer value) {
- return isDirty(1);
+ public boolean isMajorCodeDirty(java.lang.Integer value) {
+ return isDirty(0);
}
/**
* Gets the value of the 'minorCode' field.
- */
- public Integer getMinorCode() {
+ * Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. */
+ public java.lang.Integer getMinorCode() {
return minorCode;
}
/**
* Sets the value of the 'minorCode' field.
- * @param value the value to set.
+ * Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. * @param value the value to set.
*/
- public void setMinorCode(Integer value) {
+ public void setMinorCode(java.lang.Integer value) {
this.minorCode = value;
- setDirty(2);
+ setDirty(1);
}
/**
* Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. * @param value the value to set.
*/
- public boolean isMinorCodeDirty(Integer value) {
- return isDirty(2);
+ public boolean isMinorCodeDirty(java.lang.Integer value) {
+ return isDirty(1);
}
/**
* Gets the value of the 'args' field.
- */
- public java.util.List getArgs() {
+ * Optional arguments supplied to compliment and/or justify the parse status code. */
+ public java.util.List getArgs() {
return args;
}
/**
* Sets the value of the 'args' field.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the parse status code. * @param value the value to set.
*/
- public void setArgs(java.util.List value) {
+ public void setArgs(java.util.List value) {
this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value);
- setDirty(3);
+ setDirty(2);
}
/**
* Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the parse status code. * @param value the value to set.
*/
- public boolean isArgsDirty(java.util.List value) {
- return isDirty(3);
+ public boolean isArgsDirty(java.util.List value) {
+ return isDirty(2);
}
/** Creates a new ParseStatus RecordBuilder */
- public static Builder newBuilder() {
- return new Builder();
+ public static org.apache.nutch.storage.ParseStatus.Builder newBuilder() {
+ return new org.apache.nutch.storage.ParseStatus.Builder();
}
/** Creates a new ParseStatus RecordBuilder by copying an existing Builder */
- public static Builder newBuilder(Builder other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.ParseStatus.Builder newBuilder(org.apache.nutch.storage.ParseStatus.Builder other) {
+ return new org.apache.nutch.storage.ParseStatus.Builder(other);
}
/** Creates a new ParseStatus RecordBuilder by copying an existing ParseStatus instance */
- public static Builder newBuilder(ParseStatus other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.ParseStatus.Builder newBuilder(org.apache.nutch.storage.ParseStatus other) {
+ return new org.apache.nutch.storage.ParseStatus.Builder(other);
}
- private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer(
+ private static java.nio.ByteBuffer deepCopyToReadOnlyBuffer(
java.nio.ByteBuffer input) {
java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity());
int position = input.position();
@@ -227,112 +225,107 @@
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase
implements org.apache.avro.data.RecordBuilder {
- private java.nio.ByteBuffer __g__dirty;
private int majorCode;
private int minorCode;
- private java.util.List args;
+ private java.util.List args;
/** Creates a new Builder */
private Builder() {
- super(ParseStatus.SCHEMA$);
+ super(org.apache.nutch.storage.ParseStatus.SCHEMA$);
}
/** Creates a Builder by copying an existing Builder */
- private Builder(Builder other) {
+ private Builder(org.apache.nutch.storage.ParseStatus.Builder other) {
super(other);
}
/** Creates a Builder by copying an existing ParseStatus instance */
- private Builder(ParseStatus other) {
- super(ParseStatus.SCHEMA$);
- if (isValidValue(fields()[0], other.__g__dirty)) {
- this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty);
+ private Builder(org.apache.nutch.storage.ParseStatus other) {
+ super(org.apache.nutch.storage.ParseStatus.SCHEMA$);
+ if (isValidValue(fields()[0], other.majorCode)) {
+ this.majorCode = (java.lang.Integer) data().deepCopy(fields()[0].schema(), other.majorCode);
fieldSetFlags()[0] = true;
}
- if (isValidValue(fields()[1], other.majorCode)) {
- this.majorCode = (Integer) data().deepCopy(fields()[1].schema(), other.majorCode);
+ if (isValidValue(fields()[1], other.minorCode)) {
+ this.minorCode = (java.lang.Integer) data().deepCopy(fields()[1].schema(), other.minorCode);
fieldSetFlags()[1] = true;
}
- if (isValidValue(fields()[2], other.minorCode)) {
- this.minorCode = (Integer) data().deepCopy(fields()[2].schema(), other.minorCode);
+ if (isValidValue(fields()[2], other.args)) {
+ this.args = (java.util.List) data().deepCopy(fields()[2].schema(), other.args);
fieldSetFlags()[2] = true;
}
- if (isValidValue(fields()[3], other.args)) {
- this.args = (java.util.List) data().deepCopy(fields()[3].schema(), other.args);
- fieldSetFlags()[3] = true;
- }
}
/** Gets the value of the 'majorCode' field */
- public Integer getMajorCode() {
+ public java.lang.Integer getMajorCode() {
return majorCode;
}
/** Sets the value of the 'majorCode' field */
- public Builder setMajorCode(int value) {
- validate(fields()[1], value);
+ public org.apache.nutch.storage.ParseStatus.Builder setMajorCode(int value) {
+ validate(fields()[0], value);
this.majorCode = value;
- fieldSetFlags()[1] = true;
+ fieldSetFlags()[0] = true;
return this;
}
/** Checks whether the 'majorCode' field has been set */
public boolean hasMajorCode() {
- return fieldSetFlags()[1];
+ return fieldSetFlags()[0];
}
/** Clears the value of the 'majorCode' field */
- public Builder clearMajorCode() {
- fieldSetFlags()[1] = false;
+ public org.apache.nutch.storage.ParseStatus.Builder clearMajorCode() {
+ fieldSetFlags()[0] = false;
return this;
}
/** Gets the value of the 'minorCode' field */
- public Integer getMinorCode() {
+ public java.lang.Integer getMinorCode() {
return minorCode;
}
/** Sets the value of the 'minorCode' field */
- public Builder setMinorCode(int value) {
- validate(fields()[2], value);
+ public org.apache.nutch.storage.ParseStatus.Builder setMinorCode(int value) {
+ validate(fields()[1], value);
this.minorCode = value;
- fieldSetFlags()[2] = true;
+ fieldSetFlags()[1] = true;
return this;
}
/** Checks whether the 'minorCode' field has been set */
public boolean hasMinorCode() {
- return fieldSetFlags()[2];
+ return fieldSetFlags()[1];
}
/** Clears the value of the 'minorCode' field */
- public Builder clearMinorCode() {
- fieldSetFlags()[2] = false;
+ public org.apache.nutch.storage.ParseStatus.Builder clearMinorCode() {
+ fieldSetFlags()[1] = false;
return this;
}
/** Gets the value of the 'args' field */
- public java.util.List getArgs() {
+ public java.util.List getArgs() {
return args;
}
/** Sets the value of the 'args' field */
- public Builder setArgs(java.util.List value) {
- validate(fields()[3], value);
+ public org.apache.nutch.storage.ParseStatus.Builder setArgs(java.util.List value) {
+ validate(fields()[2], value);
this.args = value;
- fieldSetFlags()[3] = true;
+ fieldSetFlags()[2] = true;
return this;
}
/** Checks whether the 'args' field has been set */
public boolean hasArgs() {
- return fieldSetFlags()[3];
+ return fieldSetFlags()[2];
}
/** Clears the value of the 'args' field */
- public Builder clearArgs() {
+ public org.apache.nutch.storage.ParseStatus.Builder clearArgs() {
args = null;
- fieldSetFlags()[3] = false;
+ fieldSetFlags()[2] = false;
return this;
}
@@ -340,10 +333,9 @@
public ParseStatus build() {
try {
ParseStatus record = new ParseStatus();
- record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]);
- record.majorCode = fieldSetFlags()[1] ? this.majorCode : (Integer) defaultValue(fields()[1]);
- record.minorCode = fieldSetFlags()[2] ? this.minorCode : (Integer) defaultValue(fields()[2]);
- record.args = fieldSetFlags()[3] ? this.args : (java.util.List) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[3]));
+ record.majorCode = fieldSetFlags()[0] ? this.majorCode : (java.lang.Integer) defaultValue(fields()[0]);
+ record.minorCode = fieldSetFlags()[1] ? this.minorCode : (java.lang.Integer) defaultValue(fields()[1]);
+ record.args = fieldSetFlags()[2] ? this.args : (java.util.List) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[2]));
return record;
} catch (Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
@@ -351,7 +343,7 @@
}
}
- public Tombstone getTombstone(){
+ public ParseStatus.Tombstone getTombstone(){
return TOMBSTONE;
}
@@ -365,76 +357,77 @@
private Tombstone() { }
- /**
+ /**
* Gets the value of the 'majorCode' field.
- */
- public Integer getMajorCode() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) */
+ public java.lang.Integer getMajorCode() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'majorCode' field.
- * @param value the value to set.
+ * Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) * @param value the value to set.
*/
- public void setMajorCode(Integer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setMajorCode(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'majorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.) * @param value the value to set.
*/
- public boolean isMajorCodeDirty(Integer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isMajorCodeDirty(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'minorCode' field.
- */
- public Integer getMinorCode() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. */
+ public java.lang.Integer getMinorCode() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'minorCode' field.
- * @param value the value to set.
+ * Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. * @param value the value to set.
*/
- public void setMinorCode(Integer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setMinorCode(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'minorCode' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage. * @param value the value to set.
*/
- public boolean isMinorCodeDirty(Integer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isMinorCodeDirty(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'args' field.
- */
- public java.util.List getArgs() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Optional arguments supplied to compliment and/or justify the parse status code. */
+ public java.util.List getArgs() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'args' field.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the parse status code. * @param value the value to set.
*/
- public void setArgs(java.util.List value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setArgs(java.util.List value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the parse status code. * @param value the value to set.
*/
- public boolean isArgsDirty(java.util.List value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isArgsDirty(java.util.List value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
}
-}
\ No newline at end of file
+}
+
Index: src/java/org/apache/nutch/storage/ProtocolStatus.java
===================================================================
--- src/java/org/apache/nutch/storage/ProtocolStatus.java (revision 1650154)
+++ src/java/org/apache/nutch/storage/ProtocolStatus.java (working copy)
@@ -1,19 +1,19 @@
/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+******************************************************************************/
/**
* Autogenerated by Avro
*
@@ -24,15 +24,15 @@
import org.apache.nutch.protocol.ProtocolStatusUtils;
@SuppressWarnings("all")
+/** A nested container representing data captured from web server responses. */
public class ProtocolStatus extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"code\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"default\":0}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"namespace\":\"org.apache.nutch.storage\",\"doc\":\"A nested container representing data captured from web server responses.\",\"fields\":[{\"name\":\"code\",\"type\":\"int\",\"doc\":\"A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching.\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"doc\":\"Optional arguments supplied to compliment and/or justify the response code.\",\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"doc\":\"A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself.\",\"default\":0}]}");
/** Enum containing all data bean's fields. */
public static enum Field {
- __G__DIRTY(0, "__g__dirty"),
- CODE(1, "code"),
- ARGS(2, "args"),
- LAST_MODIFIED(3, "lastModified"),
+ CODE(0, "code"),
+ ARGS(1, "args"),
+ LAST_MODIFIED(2, "lastModified"),
;
/**
* Field's index.
@@ -71,7 +71,6 @@
};
public static final String[] _ALL_FIELDS = {
- "__g__dirty",
"code",
"args",
"lastModified",
@@ -84,20 +83,20 @@
public int getFieldsCount() {
return ProtocolStatus._ALL_FIELDS.length;
}
-
- /** Bytes used to represent weather or not a field is dirty. */
- private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[1]);
+
+ /** A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. */
private int code;
- private java.util.List args;
+ /** Optional arguments supplied to compliment and/or justify the response code. */
+ private java.util.List args;
+ /** A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. */
private long lastModified;
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
- public Object get(int field$) {
+ public java.lang.Object get(int field$) {
switch (field$) {
- case 0: return __g__dirty;
- case 1: return code;
- case 2: return args;
- case 3: return lastModified;
+ case 0: return code;
+ case 1: return args;
+ case 2: return lastModified;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -104,12 +103,11 @@
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
- public void put(int field$, Object value) {
+ public void put(int field$, java.lang.Object value) {
switch (field$) {
- case 0: __g__dirty = (java.nio.ByteBuffer)(value); break;
- case 1: code = (Integer)(value); break;
- case 2: args = (java.util.List)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break;
- case 3: lastModified = (Long)(value); break;
+ case 0: code = (java.lang.Integer)(value); break;
+ case 1: args = (java.util.List)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)value)); break;
+ case 2: lastModified = (java.lang.Long)(value); break;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -116,101 +114,92 @@
/**
* Gets the value of the 'code' field.
- */
- public Integer getCode() {
+ * A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. */
+ public java.lang.Integer getCode() {
return code;
}
/**
* Sets the value of the 'code' field.
- * @param value the value to set.
+ * A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. * @param value the value to set.
*/
- public void setCode(Integer value) {
+ public void setCode(java.lang.Integer value) {
this.code = value;
- setDirty(1);
+ setDirty(0);
}
/**
* Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. * @param value the value to set.
*/
- public boolean isCodeDirty(Integer value) {
- return isDirty(1);
+ public boolean isCodeDirty(java.lang.Integer value) {
+ return isDirty(0);
}
/**
* Gets the value of the 'args' field.
- */
- public java.util.List getArgs() {
+ * Optional arguments supplied to compliment and/or justify the response code. */
+ public java.util.List getArgs() {
return args;
}
/**
* Sets the value of the 'args' field.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the response code. * @param value the value to set.
*/
- public void setArgs(java.util.List value) {
+ public void setArgs(java.util.List value) {
this.args = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyListWrapper(value);
- setDirty(2);
+ setDirty(1);
}
/**
* Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the response code. * @param value the value to set.
*/
- public boolean isArgsDirty(java.util.List value) {
- return isDirty(2);
+ public boolean isArgsDirty(java.util.List value) {
+ return isDirty(1);
}
/**
* Gets the value of the 'lastModified' field.
- */
- public Long getLastModified() {
+ * A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. */
+ public java.lang.Long getLastModified() {
return lastModified;
}
/**
* Sets the value of the 'lastModified' field.
- * @param value the value to set.
+ * A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. * @param value the value to set.
*/
- public void setLastModified(Long value) {
+ public void setLastModified(java.lang.Long value) {
this.lastModified = value;
- setDirty(3);
+ setDirty(2);
}
/**
* Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. * @param value the value to set.
*/
- public boolean isLastModifiedDirty(Long value) {
- return isDirty(3);
+ public boolean isLastModifiedDirty(java.lang.Long value) {
+ return isDirty(2);
}
- /**
- * A convenience method which returns a successful {@link ProtocolStatus}.
- *
- * @return the {@link ProtocolStatus} value for 200 (success).
- */
- public boolean isSuccess() {
- return code == ProtocolStatusUtils.SUCCESS;
- }
-
/** Creates a new ProtocolStatus RecordBuilder */
- public static Builder newBuilder() {
- return new Builder();
+ public static org.apache.nutch.storage.ProtocolStatus.Builder newBuilder() {
+ return new org.apache.nutch.storage.ProtocolStatus.Builder();
}
/** Creates a new ProtocolStatus RecordBuilder by copying an existing Builder */
- public static Builder newBuilder(Builder other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.ProtocolStatus.Builder newBuilder(org.apache.nutch.storage.ProtocolStatus.Builder other) {
+ return new org.apache.nutch.storage.ProtocolStatus.Builder(other);
}
/** Creates a new ProtocolStatus RecordBuilder by copying an existing ProtocolStatus instance */
- public static Builder newBuilder(ProtocolStatus other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.ProtocolStatus.Builder newBuilder(org.apache.nutch.storage.ProtocolStatus other) {
+ return new org.apache.nutch.storage.ProtocolStatus.Builder(other);
}
- private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer(
+ private static java.nio.ByteBuffer deepCopyToReadOnlyBuffer(
java.nio.ByteBuffer input) {
java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity());
int position = input.position();
@@ -239,112 +228,107 @@
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase
implements org.apache.avro.data.RecordBuilder {
- private java.nio.ByteBuffer __g__dirty;
private int code;
- private java.util.List args;
+ private java.util.List args;
private long lastModified;
/** Creates a new Builder */
private Builder() {
- super(ProtocolStatus.SCHEMA$);
+ super(org.apache.nutch.storage.ProtocolStatus.SCHEMA$);
}
/** Creates a Builder by copying an existing Builder */
- private Builder(Builder other) {
+ private Builder(org.apache.nutch.storage.ProtocolStatus.Builder other) {
super(other);
}
/** Creates a Builder by copying an existing ProtocolStatus instance */
- private Builder(ProtocolStatus other) {
- super(ProtocolStatus.SCHEMA$);
- if (isValidValue(fields()[0], other.__g__dirty)) {
- this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty);
+ private Builder(org.apache.nutch.storage.ProtocolStatus other) {
+ super(org.apache.nutch.storage.ProtocolStatus.SCHEMA$);
+ if (isValidValue(fields()[0], other.code)) {
+ this.code = (java.lang.Integer) data().deepCopy(fields()[0].schema(), other.code);
fieldSetFlags()[0] = true;
}
- if (isValidValue(fields()[1], other.code)) {
- this.code = (Integer) data().deepCopy(fields()[1].schema(), other.code);
+ if (isValidValue(fields()[1], other.args)) {
+ this.args = (java.util.List) data().deepCopy(fields()[1].schema(), other.args);
fieldSetFlags()[1] = true;
}
- if (isValidValue(fields()[2], other.args)) {
- this.args = (java.util.List) data().deepCopy(fields()[2].schema(), other.args);
+ if (isValidValue(fields()[2], other.lastModified)) {
+ this.lastModified = (java.lang.Long) data().deepCopy(fields()[2].schema(), other.lastModified);
fieldSetFlags()[2] = true;
}
- if (isValidValue(fields()[3], other.lastModified)) {
- this.lastModified = (Long) data().deepCopy(fields()[3].schema(), other.lastModified);
- fieldSetFlags()[3] = true;
- }
}
/** Gets the value of the 'code' field */
- public Integer getCode() {
+ public java.lang.Integer getCode() {
return code;
}
/** Sets the value of the 'code' field */
- public Builder setCode(int value) {
- validate(fields()[1], value);
+ public org.apache.nutch.storage.ProtocolStatus.Builder setCode(int value) {
+ validate(fields()[0], value);
this.code = value;
- fieldSetFlags()[1] = true;
+ fieldSetFlags()[0] = true;
return this;
}
/** Checks whether the 'code' field has been set */
public boolean hasCode() {
- return fieldSetFlags()[1];
+ return fieldSetFlags()[0];
}
/** Clears the value of the 'code' field */
- public Builder clearCode() {
- fieldSetFlags()[1] = false;
+ public org.apache.nutch.storage.ProtocolStatus.Builder clearCode() {
+ fieldSetFlags()[0] = false;
return this;
}
/** Gets the value of the 'args' field */
- public java.util.List getArgs() {
+ public java.util.List getArgs() {
return args;
}
/** Sets the value of the 'args' field */
- public Builder setArgs(java.util.List value) {
- validate(fields()[2], value);
+ public org.apache.nutch.storage.ProtocolStatus.Builder setArgs(java.util.List value) {
+ validate(fields()[1], value);
this.args = value;
- fieldSetFlags()[2] = true;
+ fieldSetFlags()[1] = true;
return this;
}
/** Checks whether the 'args' field has been set */
public boolean hasArgs() {
- return fieldSetFlags()[2];
+ return fieldSetFlags()[1];
}
/** Clears the value of the 'args' field */
- public Builder clearArgs() {
+ public org.apache.nutch.storage.ProtocolStatus.Builder clearArgs() {
args = null;
- fieldSetFlags()[2] = false;
+ fieldSetFlags()[1] = false;
return this;
}
/** Gets the value of the 'lastModified' field */
- public Long getLastModified() {
+ public java.lang.Long getLastModified() {
return lastModified;
}
/** Sets the value of the 'lastModified' field */
- public Builder setLastModified(long value) {
- validate(fields()[3], value);
+ public org.apache.nutch.storage.ProtocolStatus.Builder setLastModified(long value) {
+ validate(fields()[2], value);
this.lastModified = value;
- fieldSetFlags()[3] = true;
+ fieldSetFlags()[2] = true;
return this;
}
/** Checks whether the 'lastModified' field has been set */
public boolean hasLastModified() {
- return fieldSetFlags()[3];
+ return fieldSetFlags()[2];
}
/** Clears the value of the 'lastModified' field */
- public Builder clearLastModified() {
- fieldSetFlags()[3] = false;
+ public org.apache.nutch.storage.ProtocolStatus.Builder clearLastModified() {
+ fieldSetFlags()[2] = false;
return this;
}
@@ -352,10 +336,9 @@
public ProtocolStatus build() {
try {
ProtocolStatus record = new ProtocolStatus();
- record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[1]);
- record.code = fieldSetFlags()[1] ? this.code : (Integer) defaultValue(fields()[1]);
- record.args = fieldSetFlags()[2] ? this.args : (java.util.List) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[2]));
- record.lastModified = fieldSetFlags()[3] ? this.lastModified : (Long) defaultValue(fields()[3]);
+ record.code = fieldSetFlags()[0] ? this.code : (java.lang.Integer) defaultValue(fields()[0]);
+ record.args = fieldSetFlags()[1] ? this.args : (java.util.List) new org.apache.gora.persistency.impl.DirtyListWrapper((java.util.List)defaultValue(fields()[1]));
+ record.lastModified = fieldSetFlags()[2] ? this.lastModified : (java.lang.Long) defaultValue(fields()[2]);
return record;
} catch (Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
@@ -363,7 +346,7 @@
}
}
- public Tombstone getTombstone(){
+ public ProtocolStatus.Tombstone getTombstone(){
return TOMBSTONE;
}
@@ -371,6 +354,15 @@
return newBuilder().build();
}
+ // TODO NUTCH-1709 Generated classes o.a.n.storage.Host and o.a.n.storage.ProtocolStatus contain methods not defined in source .avsc
+ /**
+ * A convenience method which returns a successful {@link ProtocolStatus}.
+ * @return the {@link ProtocolStatus} value for 200 (success).
+ */
+ public boolean isSuccess() {
+ return code == ProtocolStatusUtils.SUCCESS;
+ }
+
private static final Tombstone TOMBSTONE = new Tombstone();
public static final class Tombstone extends ProtocolStatus implements org.apache.gora.persistency.Tombstone {
@@ -377,76 +369,77 @@
private Tombstone() { }
- /**
+ /**
* Gets the value of the 'code' field.
- */
- public Integer getCode() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. */
+ public java.lang.Integer getCode() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'code' field.
- * @param value the value to set.
+ * A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. * @param value the value to set.
*/
- public void setCode(Integer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setCode(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'code' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching. * @param value the value to set.
*/
- public boolean isCodeDirty(Integer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isCodeDirty(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'args' field.
- */
- public java.util.List getArgs() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Optional arguments supplied to compliment and/or justify the response code. */
+ public java.util.List getArgs() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'args' field.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the response code. * @param value the value to set.
*/
- public void setArgs(java.util.List value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setArgs(java.util.List value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'args' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Optional arguments supplied to compliment and/or justify the response code. * @param value the value to set.
*/
- public boolean isArgsDirty(java.util.List value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isArgsDirty(java.util.List value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'lastModified' field.
- */
- public Long getLastModified() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. */
+ public java.lang.Long getLastModified() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'lastModified' field.
- * @param value the value to set.
+ * A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. * @param value the value to set.
*/
- public void setLastModified(Long value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setLastModified(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'lastModified' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself. * @param value the value to set.
*/
- public boolean isLastModifiedDirty(Long value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isLastModifiedDirty(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
}
-}
\ No newline at end of file
+}
+
Index: src/java/org/apache/nutch/storage/WebPage.java
===================================================================
--- src/java/org/apache/nutch/storage/WebPage.java (revision 1650154)
+++ src/java/org/apache/nutch/storage/WebPage.java (working copy)
@@ -1,19 +1,19 @@
/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+******************************************************************************/
/**
* Autogenerated by Avro
*
@@ -21,36 +21,36 @@
*/
package org.apache.nutch.storage;
@SuppressWarnings("all")
+/** WebPage is the primary data structure in Nutch representing crawl data for a given WebPage at some point in time */
public class WebPage extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.nutch.storage\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AAAAAA==\"},{\"name\":\"baseUrl\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"status\",\"type\":\"int\",\"default\":0},{\"name\":\"fetchTime\",\"type\":\"long\",\"default\":0},{\"name\":\"prevFetchTime\",\"type\":\"long\",\"default\":0},{\"name\":\"fetchInterval\",\"type\":\"int\",\"default\":0},{\"name\":\"retriesSinceFetch\",\"type\":\"int\",\"default\":0},{\"name\":\"modifiedTime\",\"type\":\"long\",\"default\":0},{\"name\":\"prevModifiedTime\",\"type\":\"long\",\"default\":0},{\"name\":\"protocolStatus\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"code\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"default\":0}]}],\"default\":null},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"contentType\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"prevSignature\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"signature\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"title\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"text\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"parseStatus\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"ParseStatus\",\"fields\":[{\"name\":\"__g__dirty\",\"type\":\"bytes\",\"doc\":\"Bytes used to represent weather or not a field is dirty.\",\"default\":\"AA==\"},{\"name\":\"majorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":[]}]}],\"default\":null},{\"name\":\"score\",\"type\":\"float\",\"default\":0},{\"name\":\"reprUrl\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"headers\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"markers\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"bytes\"]},\"default\":{}},{\"name\":\"batchId\",\"type\":[\"null\",\"string\"],\"default\":null}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.nutch.storage\",\"doc\":\"WebPage is the primary data structure in Nutch representing crawl data for a given WebPage at some point in time\",\"fields\":[{\"name\":\"baseUrl\",\"type\":[\"null\",\"string\"],\"doc\":\"The original associated with this WebPage.\",\"default\":null},{\"name\":\"status\",\"type\":\"int\",\"doc\":\"A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified\",\"default\":0},{\"name\":\"fetchTime\",\"type\":\"long\",\"doc\":\"The system time in milliseconds for when the page was fetched.\",\"default\":0},{\"name\":\"prevFetchTime\",\"type\":\"long\",\"doc\":\"The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation\",\"default\":0},{\"name\":\"fetchInterval\",\"type\":\"int\",\"doc\":\"The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented.\",\"default\":0},{\"name\":\"retriesSinceFetch\",\"type\":\"int\",\"doc\":\"The number of retried attempts at fetching the WebPage since it was last successfully fetched.\",\"default\":0},{\"name\":\"modifiedTime\",\"type\":\"long\",\"doc\":\"The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage.\",\"default\":0},{\"name\":\"prevModifiedTime\",\"type\":\"long\",\"doc\":\"The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage.\",\"default\":0},{\"name\":\"protocolStatus\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"ProtocolStatus\",\"doc\":\"A nested container representing data captured from web server responses.\",\"fields\":[{\"name\":\"code\",\"type\":\"int\",\"doc\":\"A protocol response code which can be one of SUCCESS - content was retrieved without errors, FAILED - Content was not retrieved. Any further errors may be indicated in args, PROTO_NOT_FOUND - This protocol was not found. Application may attempt to retry later, GONE - Resource is gone, MOVED - Resource has moved permanently. New url should be found in args, TEMP_MOVED - Resource has moved temporarily. New url should be found in args., NOTFOUND - Resource was not found, RETRY - Temporary failure. Application may retry immediately., EXCEPTION - Unspecified exception occured. Further information may be provided in args., ACCESS_DENIED - Access denied - authorization required, but missing/incorrect., ROBOTS_DENIED - Access denied by robots.txt rules., REDIR_EXCEEDED - Too many redirects., NOTFETCHING - Not fetching., NOTMODIFIED - Unchanged since the last fetch., WOULDBLOCK - Request was refused by protocol plugins, because it would block. The expected number of milliseconds to wait before retry may be provided in args., BLOCKED - Thread was blocked http.max.delays times during fetching.\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"doc\":\"Optional arguments supplied to compliment and/or justify the response code.\",\"default\":[]},{\"name\":\"lastModified\",\"type\":\"long\",\"doc\":\"A server reponse indicating when this page was last modified, this can be unreliable at times hence this is used as a default fall back value for the preferred 'modifiedTime' and 'preModifiedTime' obtained from the WebPage itself.\",\"default\":0}]}],\"default\":null},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"],\"doc\":\"The entire raw document content e.g. raw XHTML\",\"default\":null},{\"name\":\"contentType\",\"type\":[\"null\",\"string\"],\"doc\":\"The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used.\",\"default\":null},{\"name\":\"prevSignature\",\"type\":[\"null\",\"bytes\"],\"doc\":\"An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints.\",\"default\":null},{\"name\":\"signature\",\"type\":[\"null\",\"bytes\"],\"doc\":\"An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time.\",\"default\":null},{\"name\":\"title\",\"type\":[\"null\",\"string\"],\"doc\":\"The title of the WebPage.\",\"default\":null},{\"name\":\"text\",\"type\":[\"null\",\"string\"],\"doc\":\"The textual content of the WebPage devoid from native markup.\",\"default\":null},{\"name\":\"parseStatus\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"ParseStatus\",\"doc\":\"A nested container representing parse status data captured from invocation of parsers on fetch of a WebPage\",\"fields\":[{\"name\":\"majorCode\",\"type\":\"int\",\"doc\":\"Major parsing status' including NOTPARSED (Parsing was not performed), SUCCESS (Parsing succeeded), FAILED (General failure. There may be a more specific error message in arguments.)\",\"default\":0},{\"name\":\"minorCode\",\"type\":\"int\",\"doc\":\"Minor parsing status' including SUCCESS_OK - Successful parse devoid of anomalies or issues, SUCCESS_REDIRECT - Parsed content contains a directive to redirect to another URL. The target URL can be retrieved from the arguments., FAILED_EXCEPTION - Parsing failed. An Exception occured which may be retrieved from the arguments., FAILED_TRUNCATED - Parsing failed. Content was truncated, but the parser cannot handle incomplete content., FAILED_INVALID_FORMAT - Parsing failed. Invalid format e.g. the content may be corrupted or of wrong type., FAILED_MISSING_PARTS - Parsing failed. Other related parts of the content are needed to complete parsing. The list of URLs to missing parts may be provided in arguments. The Fetcher may decide to fetch these parts at once, then put them into Content.metadata, and supply them for re-parsing., FAILED_MISING_CONTENT - Parsing failed. There was no content to be parsed - probably caused by errors at protocol stage.\",\"default\":0},{\"name\":\"args\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"doc\":\"Optional arguments supplied to compliment and/or justify the parse status code.\",\"default\":[]}]}],\"default\":null},{\"name\":\"score\",\"type\":\"float\",\"doc\":\"A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics.\",\"default\":0},{\"name\":\"reprUrl\",\"type\":[\"null\",\"string\"],\"doc\":\"In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler\",\"default\":null},{\"name\":\"headers\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"doc\":\"Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION.\",\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"doc\":\"Embedded hyperlinks which direct outside of the current domain.\",\"default\":{}},{\"name\":\"inlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"doc\":\"Embedded hyperlinks which link to pages within the current domain.\",\"default\":{}},{\"name\":\"markers\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"doc\":\"Markers flags which represent user and machine decisions which have affected influenced a WebPage's current state. Markers can be system specific and user machine driven in nature. They are assigned to a WebPage on a job-by-job basis and thier values indicative of what actions should be associated with a WebPage.\",\"default\":{}},{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"bytes\"]},\"doc\":\"A multi-valued metadata container used for storing everything from structured WebPage characterists, to ad-hoc extraction and metadata augmentation for any given WebPage.\",\"default\":{}},{\"name\":\"batchId\",\"type\":[\"null\",\"string\"],\"doc\":\"A batchId that this WebPage is assigned to. WebPage's are fetched in batches, called fetchlists. Pages are partitioned but can always be associated and fetched alongside pages of similar value (within a crawl cycle) based on batchId.\",\"default\":null}]}");
/** Enum containing all data bean's fields. */
public static enum Field {
- __G__DIRTY(0, "__g__dirty"),
- BASE_URL(1, "baseUrl"),
- STATUS(2, "status"),
- FETCH_TIME(3, "fetchTime"),
- PREV_FETCH_TIME(4, "prevFetchTime"),
- FETCH_INTERVAL(5, "fetchInterval"),
- RETRIES_SINCE_FETCH(6, "retriesSinceFetch"),
- MODIFIED_TIME(7, "modifiedTime"),
- PREV_MODIFIED_TIME(8, "prevModifiedTime"),
- PROTOCOL_STATUS(9, "protocolStatus"),
- CONTENT(10, "content"),
- CONTENT_TYPE(11, "contentType"),
- PREV_SIGNATURE(12, "prevSignature"),
- SIGNATURE(13, "signature"),
- TITLE(14, "title"),
- TEXT(15, "text"),
- PARSE_STATUS(16, "parseStatus"),
- SCORE(17, "score"),
- REPR_URL(18, "reprUrl"),
- HEADERS(19, "headers"),
- OUTLINKS(20, "outlinks"),
- INLINKS(21, "inlinks"),
- MARKERS(22, "markers"),
- METADATA(23, "metadata"),
- BATCH_ID(24, "batchId"),
+ BASE_URL(0, "baseUrl"),
+ STATUS(1, "status"),
+ FETCH_TIME(2, "fetchTime"),
+ PREV_FETCH_TIME(3, "prevFetchTime"),
+ FETCH_INTERVAL(4, "fetchInterval"),
+ RETRIES_SINCE_FETCH(5, "retriesSinceFetch"),
+ MODIFIED_TIME(6, "modifiedTime"),
+ PREV_MODIFIED_TIME(7, "prevModifiedTime"),
+ PROTOCOL_STATUS(8, "protocolStatus"),
+ CONTENT(9, "content"),
+ CONTENT_TYPE(10, "contentType"),
+ PREV_SIGNATURE(11, "prevSignature"),
+ SIGNATURE(12, "signature"),
+ TITLE(13, "title"),
+ TEXT(14, "text"),
+ PARSE_STATUS(15, "parseStatus"),
+ SCORE(16, "score"),
+ REPR_URL(17, "reprUrl"),
+ HEADERS(18, "headers"),
+ OUTLINKS(19, "outlinks"),
+ INLINKS(20, "inlinks"),
+ MARKERS(21, "markers"),
+ METADATA(22, "metadata"),
+ BATCH_ID(23, "batchId"),
;
/**
* Field's index.
@@ -89,7 +89,6 @@
};
public static final String[] _ALL_FIELDS = {
- "__g__dirty",
"baseUrl",
"status",
"fetchTime",
@@ -124,62 +123,80 @@
return WebPage._ALL_FIELDS.length;
}
-
- /** Bytes used to represent weather or not a field is dirty. */
- private java.nio.ByteBuffer __g__dirty = java.nio.ByteBuffer.wrap(new byte[4]);
- private CharSequence baseUrl;
+ /** The original associated with this WebPage. */
+ private java.lang.CharSequence baseUrl;
+ /** A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified */
private int status;
+ /** The system time in milliseconds for when the page was fetched. */
private long fetchTime;
+ /** The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation */
private long prevFetchTime;
+ /** The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. */
private int fetchInterval;
+ /** The number of retried attempts at fetching the WebPage since it was last successfully fetched. */
private int retriesSinceFetch;
+ /** The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. */
private long modifiedTime;
+ /** The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. */
private long prevModifiedTime;
- private ProtocolStatus protocolStatus;
+ private org.apache.nutch.storage.ProtocolStatus protocolStatus;
+ /** The entire raw document content e.g. raw XHTML */
private java.nio.ByteBuffer content;
- private CharSequence contentType;
+ /** The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. */
+ private java.lang.CharSequence contentType;
+ /** An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. */
private java.nio.ByteBuffer prevSignature;
+ /** An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. */
private java.nio.ByteBuffer signature;
- private CharSequence title;
- private CharSequence text;
- private ParseStatus parseStatus;
+ /** The title of the WebPage. */
+ private java.lang.CharSequence title;
+ /** The textual content of the WebPage devoid from native markup. */
+ private java.lang.CharSequence text;
+ private org.apache.nutch.storage.ParseStatus parseStatus;
+ /** A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. */
private float score;
- private CharSequence reprUrl;
- private java.util.Map headers;
- private java.util.Map outlinks;
- private java.util.Map inlinks;
- private java.util.Map markers;
- private java.util.Map metadata;
- private CharSequence batchId;
+ /** In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler */
+ private java.lang.CharSequence reprUrl;
+ /** Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. */
+ private java.util.Map headers;
+ /** Embedded hyperlinks which direct outside of the current domain. */
+ private java.util.Map outlinks;
+ /** Embedded hyperlinks which link to pages within the current domain. */
+ private java.util.Map inlinks;
+ /** Markers flags which represent user and machine decisions which have affected influenced a WebPage's current state. Markers can be system specific and user machine driven in nature. They are assigned to a WebPage on a job-by-job basis and thier values indicative of what actions should be associated with a WebPage. */
+ private java.util.Map markers;
+ /** A multi-valued metadata container used for storing everything from structured WebPage characterists, to ad-hoc extraction and metadata augmentation for any given WebPage. */
+ private java.util.Map metadata;
+ /** A batchId that this WebPage is assigned to. WebPage's are fetched in batches, called fetchlists. Pages are partitioned but can always be associated and fetched alongside pages of similar value (within a crawl cycle) based on batchId. */
+ private java.lang.CharSequence batchId;
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
- public Object get(int field$) {
+ public java.lang.Object get(int field$) {
switch (field$) {
- case 0: return __g__dirty;
- case 1: return baseUrl;
- case 2: return status;
- case 3: return fetchTime;
- case 4: return prevFetchTime;
- case 5: return fetchInterval;
- case 6: return retriesSinceFetch;
- case 7: return modifiedTime;
- case 8: return prevModifiedTime;
- case 9: return protocolStatus;
- case 10: return content;
- case 11: return contentType;
- case 12: return prevSignature;
- case 13: return signature;
- case 14: return title;
- case 15: return text;
- case 16: return parseStatus;
- case 17: return score;
- case 18: return reprUrl;
- case 19: return headers;
- case 20: return outlinks;
- case 21: return inlinks;
- case 22: return markers;
- case 23: return metadata;
- case 24: return batchId;
+ case 0: return baseUrl;
+ case 1: return status;
+ case 2: return fetchTime;
+ case 3: return prevFetchTime;
+ case 4: return fetchInterval;
+ case 5: return retriesSinceFetch;
+ case 6: return modifiedTime;
+ case 7: return prevModifiedTime;
+ case 8: return protocolStatus;
+ case 9: return content;
+ case 10: return contentType;
+ case 11: return prevSignature;
+ case 12: return signature;
+ case 13: return title;
+ case 14: return text;
+ case 15: return parseStatus;
+ case 16: return score;
+ case 17: return reprUrl;
+ case 18: return headers;
+ case 19: return outlinks;
+ case 20: return inlinks;
+ case 21: return markers;
+ case 22: return metadata;
+ case 23: return batchId;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -186,33 +203,32 @@
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
- public void put(int field$, Object value) {
+ public void put(int field$, java.lang.Object value) {
switch (field$) {
- case 0: __g__dirty = (java.nio.ByteBuffer)(value); break;
- case 1: baseUrl = (CharSequence)(value); break;
- case 2: status = (Integer)(value); break;
- case 3: fetchTime = (Long)(value); break;
- case 4: prevFetchTime = (Long)(value); break;
- case 5: fetchInterval = (Integer)(value); break;
- case 6: retriesSinceFetch = (Integer)(value); break;
- case 7: modifiedTime = (Long)(value); break;
- case 8: prevModifiedTime = (Long)(value); break;
- case 9: protocolStatus = (ProtocolStatus)(value); break;
- case 10: content = (java.nio.ByteBuffer)(value); break;
- case 11: contentType = (CharSequence)(value); break;
- case 12: prevSignature = (java.nio.ByteBuffer)(value); break;
- case 13: signature = (java.nio.ByteBuffer)(value); break;
- case 14: title = (CharSequence)(value); break;
- case 15: text = (CharSequence)(value); break;
- case 16: parseStatus = (ParseStatus)(value); break;
- case 17: score = (Float)(value); break;
- case 18: reprUrl = (CharSequence)(value); break;
- case 19: headers = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 20: outlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 21: inlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 22: markers = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 23: metadata = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
- case 24: batchId = (CharSequence)(value); break;
+ case 0: baseUrl = (java.lang.CharSequence)(value); break;
+ case 1: status = (java.lang.Integer)(value); break;
+ case 2: fetchTime = (java.lang.Long)(value); break;
+ case 3: prevFetchTime = (java.lang.Long)(value); break;
+ case 4: fetchInterval = (java.lang.Integer)(value); break;
+ case 5: retriesSinceFetch = (java.lang.Integer)(value); break;
+ case 6: modifiedTime = (java.lang.Long)(value); break;
+ case 7: prevModifiedTime = (java.lang.Long)(value); break;
+ case 8: protocolStatus = (org.apache.nutch.storage.ProtocolStatus)(value); break;
+ case 9: content = (java.nio.ByteBuffer)(value); break;
+ case 10: contentType = (java.lang.CharSequence)(value); break;
+ case 11: prevSignature = (java.nio.ByteBuffer)(value); break;
+ case 12: signature = (java.nio.ByteBuffer)(value); break;
+ case 13: title = (java.lang.CharSequence)(value); break;
+ case 14: text = (java.lang.CharSequence)(value); break;
+ case 15: parseStatus = (org.apache.nutch.storage.ParseStatus)(value); break;
+ case 16: score = (java.lang.Float)(value); break;
+ case 17: reprUrl = (java.lang.CharSequence)(value); break;
+ case 18: headers = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 19: outlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 20: inlinks = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 21: markers = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 22: metadata = (java.util.Map)((value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)value)); break;
+ case 23: batchId = (java.lang.CharSequence)(value); break;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
@@ -219,200 +235,200 @@
/**
* Gets the value of the 'baseUrl' field.
- */
- public CharSequence getBaseUrl() {
+ * The original associated with this WebPage. */
+ public java.lang.CharSequence getBaseUrl() {
return baseUrl;
}
/**
* Sets the value of the 'baseUrl' field.
- * @param value the value to set.
+ * The original associated with this WebPage. * @param value the value to set.
*/
- public void setBaseUrl(CharSequence value) {
+ public void setBaseUrl(java.lang.CharSequence value) {
this.baseUrl = value;
- setDirty(1);
+ setDirty(0);
}
/**
* Checks the dirty status of the 'baseUrl' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The original associated with this WebPage. * @param value the value to set.
*/
- public boolean isBaseUrlDirty(CharSequence value) {
- return isDirty(1);
+ public boolean isBaseUrlDirty(java.lang.CharSequence value) {
+ return isDirty(0);
}
/**
* Gets the value of the 'status' field.
- */
- public Integer getStatus() {
+ * A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified */
+ public java.lang.Integer getStatus() {
return status;
}
/**
* Sets the value of the 'status' field.
- * @param value the value to set.
+ * A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified * @param value the value to set.
*/
- public void setStatus(Integer value) {
+ public void setStatus(java.lang.Integer value) {
this.status = value;
- setDirty(2);
+ setDirty(1);
}
/**
* Checks the dirty status of the 'status' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified * @param value the value to set.
*/
- public boolean isStatusDirty(Integer value) {
- return isDirty(2);
+ public boolean isStatusDirty(java.lang.Integer value) {
+ return isDirty(1);
}
/**
* Gets the value of the 'fetchTime' field.
- */
- public Long getFetchTime() {
+ * The system time in milliseconds for when the page was fetched. */
+ public java.lang.Long getFetchTime() {
return fetchTime;
}
/**
* Sets the value of the 'fetchTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was fetched. * @param value the value to set.
*/
- public void setFetchTime(Long value) {
+ public void setFetchTime(java.lang.Long value) {
this.fetchTime = value;
- setDirty(3);
+ setDirty(2);
}
/**
* Checks the dirty status of the 'fetchTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was fetched. * @param value the value to set.
*/
- public boolean isFetchTimeDirty(Long value) {
- return isDirty(3);
+ public boolean isFetchTimeDirty(java.lang.Long value) {
+ return isDirty(2);
}
/**
* Gets the value of the 'prevFetchTime' field.
- */
- public Long getPrevFetchTime() {
+ * The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation */
+ public java.lang.Long getPrevFetchTime() {
return prevFetchTime;
}
/**
* Sets the value of the 'prevFetchTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation * @param value the value to set.
*/
- public void setPrevFetchTime(Long value) {
+ public void setPrevFetchTime(java.lang.Long value) {
this.prevFetchTime = value;
- setDirty(4);
+ setDirty(3);
}
/**
* Checks the dirty status of the 'prevFetchTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation * @param value the value to set.
*/
- public boolean isPrevFetchTimeDirty(Long value) {
- return isDirty(4);
+ public boolean isPrevFetchTimeDirty(java.lang.Long value) {
+ return isDirty(3);
}
/**
* Gets the value of the 'fetchInterval' field.
- */
- public Integer getFetchInterval() {
+ * The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. */
+ public java.lang.Integer getFetchInterval() {
return fetchInterval;
}
/**
* Sets the value of the 'fetchInterval' field.
- * @param value the value to set.
+ * The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. * @param value the value to set.
*/
- public void setFetchInterval(Integer value) {
+ public void setFetchInterval(java.lang.Integer value) {
this.fetchInterval = value;
- setDirty(5);
+ setDirty(4);
}
/**
* Checks the dirty status of the 'fetchInterval' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. * @param value the value to set.
*/
- public boolean isFetchIntervalDirty(Integer value) {
- return isDirty(5);
+ public boolean isFetchIntervalDirty(java.lang.Integer value) {
+ return isDirty(4);
}
/**
* Gets the value of the 'retriesSinceFetch' field.
- */
- public Integer getRetriesSinceFetch() {
+ * The number of retried attempts at fetching the WebPage since it was last successfully fetched. */
+ public java.lang.Integer getRetriesSinceFetch() {
return retriesSinceFetch;
}
/**
* Sets the value of the 'retriesSinceFetch' field.
- * @param value the value to set.
+ * The number of retried attempts at fetching the WebPage since it was last successfully fetched. * @param value the value to set.
*/
- public void setRetriesSinceFetch(Integer value) {
+ public void setRetriesSinceFetch(java.lang.Integer value) {
this.retriesSinceFetch = value;
- setDirty(6);
+ setDirty(5);
}
/**
* Checks the dirty status of the 'retriesSinceFetch' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The number of retried attempts at fetching the WebPage since it was last successfully fetched. * @param value the value to set.
*/
- public boolean isRetriesSinceFetchDirty(Integer value) {
- return isDirty(6);
+ public boolean isRetriesSinceFetchDirty(java.lang.Integer value) {
+ return isDirty(5);
}
/**
* Gets the value of the 'modifiedTime' field.
- */
- public Long getModifiedTime() {
+ * The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. */
+ public java.lang.Long getModifiedTime() {
return modifiedTime;
}
/**
* Sets the value of the 'modifiedTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. * @param value the value to set.
*/
- public void setModifiedTime(Long value) {
+ public void setModifiedTime(java.lang.Long value) {
this.modifiedTime = value;
- setDirty(7);
+ setDirty(6);
}
/**
* Checks the dirty status of the 'modifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. * @param value the value to set.
*/
- public boolean isModifiedTimeDirty(Long value) {
- return isDirty(7);
+ public boolean isModifiedTimeDirty(java.lang.Long value) {
+ return isDirty(6);
}
/**
* Gets the value of the 'prevModifiedTime' field.
- */
- public Long getPrevModifiedTime() {
+ * The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. */
+ public java.lang.Long getPrevModifiedTime() {
return prevModifiedTime;
}
/**
* Sets the value of the 'prevModifiedTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. * @param value the value to set.
*/
- public void setPrevModifiedTime(Long value) {
+ public void setPrevModifiedTime(java.lang.Long value) {
this.prevModifiedTime = value;
- setDirty(8);
+ setDirty(7);
}
/**
* Checks the dirty status of the 'prevModifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. * @param value the value to set.
*/
- public boolean isPrevModifiedTimeDirty(Long value) {
- return isDirty(8);
+ public boolean isPrevModifiedTimeDirty(java.lang.Long value) {
+ return isDirty(7);
}
/**
* Gets the value of the 'protocolStatus' field.
*/
- public ProtocolStatus getProtocolStatus() {
+ public org.apache.nutch.storage.ProtocolStatus getProtocolStatus() {
return protocolStatus;
}
@@ -420,9 +436,9 @@
* Sets the value of the 'protocolStatus' field.
* @param value the value to set.
*/
- public void setProtocolStatus(ProtocolStatus value) {
+ public void setProtocolStatus(org.apache.nutch.storage.ProtocolStatus value) {
this.protocolStatus = value;
- setDirty(9);
+ setDirty(8);
}
/**
@@ -429,13 +445,13 @@
* Checks the dirty status of the 'protocolStatus' field. A field is dirty if it represents a change that has not yet been written to the database.
* @param value the value to set.
*/
- public boolean isProtocolStatusDirty(ProtocolStatus value) {
- return isDirty(9);
+ public boolean isProtocolStatusDirty(org.apache.nutch.storage.ProtocolStatus value) {
+ return isDirty(8);
}
/**
* Gets the value of the 'content' field.
- */
+ * The entire raw document content e.g. raw XHTML */
public java.nio.ByteBuffer getContent() {
return content;
}
@@ -442,48 +458,48 @@
/**
* Sets the value of the 'content' field.
- * @param value the value to set.
+ * The entire raw document content e.g. raw XHTML * @param value the value to set.
*/
public void setContent(java.nio.ByteBuffer value) {
this.content = value;
- setDirty(10);
+ setDirty(9);
}
/**
* Checks the dirty status of the 'content' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The entire raw document content e.g. raw XHTML * @param value the value to set.
*/
public boolean isContentDirty(java.nio.ByteBuffer value) {
- return isDirty(10);
+ return isDirty(9);
}
/**
* Gets the value of the 'contentType' field.
- */
- public CharSequence getContentType() {
+ * The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. */
+ public java.lang.CharSequence getContentType() {
return contentType;
}
/**
* Sets the value of the 'contentType' field.
- * @param value the value to set.
+ * The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. * @param value the value to set.
*/
- public void setContentType(CharSequence value) {
+ public void setContentType(java.lang.CharSequence value) {
this.contentType = value;
- setDirty(11);
+ setDirty(10);
}
/**
* Checks the dirty status of the 'contentType' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. * @param value the value to set.
*/
- public boolean isContentTypeDirty(CharSequence value) {
- return isDirty(11);
+ public boolean isContentTypeDirty(java.lang.CharSequence value) {
+ return isDirty(10);
}
/**
* Gets the value of the 'prevSignature' field.
- */
+ * An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. */
public java.nio.ByteBuffer getPrevSignature() {
return prevSignature;
}
@@ -490,24 +506,24 @@
/**
* Sets the value of the 'prevSignature' field.
- * @param value the value to set.
+ * An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. * @param value the value to set.
*/
public void setPrevSignature(java.nio.ByteBuffer value) {
this.prevSignature = value;
- setDirty(12);
+ setDirty(11);
}
/**
* Checks the dirty status of the 'prevSignature' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. * @param value the value to set.
*/
public boolean isPrevSignatureDirty(java.nio.ByteBuffer value) {
- return isDirty(12);
+ return isDirty(11);
}
/**
* Gets the value of the 'signature' field.
- */
+ * An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. */
public java.nio.ByteBuffer getSignature() {
return signature;
}
@@ -514,73 +530,73 @@
/**
* Sets the value of the 'signature' field.
- * @param value the value to set.
+ * An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. * @param value the value to set.
*/
public void setSignature(java.nio.ByteBuffer value) {
this.signature = value;
- setDirty(13);
+ setDirty(12);
}
/**
* Checks the dirty status of the 'signature' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. * @param value the value to set.
*/
public boolean isSignatureDirty(java.nio.ByteBuffer value) {
- return isDirty(13);
+ return isDirty(12);
}
/**
* Gets the value of the 'title' field.
- */
- public CharSequence getTitle() {
+ * The title of the WebPage. */
+ public java.lang.CharSequence getTitle() {
return title;
}
/**
* Sets the value of the 'title' field.
- * @param value the value to set.
+ * The title of the WebPage. * @param value the value to set.
*/
- public void setTitle(CharSequence value) {
+ public void setTitle(java.lang.CharSequence value) {
this.title = value;
- setDirty(14);
+ setDirty(13);
}
/**
* Checks the dirty status of the 'title' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The title of the WebPage. * @param value the value to set.
*/
- public boolean isTitleDirty(CharSequence value) {
- return isDirty(14);
+ public boolean isTitleDirty(java.lang.CharSequence value) {
+ return isDirty(13);
}
/**
* Gets the value of the 'text' field.
- */
- public CharSequence getText() {
+ * The textual content of the WebPage devoid from native markup. */
+ public java.lang.CharSequence getText() {
return text;
}
/**
* Sets the value of the 'text' field.
- * @param value the value to set.
+ * The textual content of the WebPage devoid from native markup. * @param value the value to set.
*/
- public void setText(CharSequence value) {
+ public void setText(java.lang.CharSequence value) {
this.text = value;
- setDirty(15);
+ setDirty(14);
}
/**
* Checks the dirty status of the 'text' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The textual content of the WebPage devoid from native markup. * @param value the value to set.
*/
- public boolean isTextDirty(CharSequence value) {
- return isDirty(15);
+ public boolean isTextDirty(java.lang.CharSequence value) {
+ return isDirty(14);
}
/**
* Gets the value of the 'parseStatus' field.
*/
- public ParseStatus getParseStatus() {
+ public org.apache.nutch.storage.ParseStatus getParseStatus() {
return parseStatus;
}
@@ -588,9 +604,9 @@
* Sets the value of the 'parseStatus' field.
* @param value the value to set.
*/
- public void setParseStatus(ParseStatus value) {
+ public void setParseStatus(org.apache.nutch.storage.ParseStatus value) {
this.parseStatus = value;
- setDirty(16);
+ setDirty(15);
}
/**
@@ -597,218 +613,218 @@
* Checks the dirty status of the 'parseStatus' field. A field is dirty if it represents a change that has not yet been written to the database.
* @param value the value to set.
*/
- public boolean isParseStatusDirty(ParseStatus value) {
- return isDirty(16);
+ public boolean isParseStatusDirty(org.apache.nutch.storage.ParseStatus value) {
+ return isDirty(15);
}
/**
* Gets the value of the 'score' field.
- */
- public Float getScore() {
+ * A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. */
+ public java.lang.Float getScore() {
return score;
}
/**
* Sets the value of the 'score' field.
- * @param value the value to set.
+ * A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. * @param value the value to set.
*/
- public void setScore(Float value) {
+ public void setScore(java.lang.Float value) {
this.score = value;
- setDirty(17);
+ setDirty(16);
}
/**
* Checks the dirty status of the 'score' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. * @param value the value to set.
*/
- public boolean isScoreDirty(Float value) {
- return isDirty(17);
+ public boolean isScoreDirty(java.lang.Float value) {
+ return isDirty(16);
}
/**
* Gets the value of the 'reprUrl' field.
- */
- public CharSequence getReprUrl() {
+ * In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler */
+ public java.lang.CharSequence getReprUrl() {
return reprUrl;
}
/**
* Sets the value of the 'reprUrl' field.
- * @param value the value to set.
+ * In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler * @param value the value to set.
*/
- public void setReprUrl(CharSequence value) {
+ public void setReprUrl(java.lang.CharSequence value) {
this.reprUrl = value;
- setDirty(18);
+ setDirty(17);
}
/**
* Checks the dirty status of the 'reprUrl' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler * @param value the value to set.
*/
- public boolean isReprUrlDirty(CharSequence value) {
- return isDirty(18);
+ public boolean isReprUrlDirty(java.lang.CharSequence value) {
+ return isDirty(17);
}
/**
* Gets the value of the 'headers' field.
- */
- public java.util.Map getHeaders() {
+ * Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. */
+ public java.util.Map getHeaders() {
return headers;
}
/**
* Sets the value of the 'headers' field.
- * @param value the value to set.
+ * Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. * @param value the value to set.
*/
- public void setHeaders(java.util.Map value) {
+ public void setHeaders(java.util.Map value) {
this.headers = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(19);
+ setDirty(18);
}
/**
* Checks the dirty status of the 'headers' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. * @param value the value to set.
*/
- public boolean isHeadersDirty(java.util.Map value) {
- return isDirty(19);
+ public boolean isHeadersDirty(java.util.Map value) {
+ return isDirty(18);
}
/**
* Gets the value of the 'outlinks' field.
- */
- public java.util.Map getOutlinks() {
+ * Embedded hyperlinks which direct outside of the current domain. */
+ public java.util.Map getOutlinks() {
return outlinks;
}
/**
* Sets the value of the 'outlinks' field.
- * @param value the value to set.
+ * Embedded hyperlinks which direct outside of the current domain. * @param value the value to set.
*/
- public void setOutlinks(java.util.Map value) {
+ public void setOutlinks(java.util.Map value) {
this.outlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(20);
+ setDirty(19);
}
/**
* Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Embedded hyperlinks which direct outside of the current domain. * @param value the value to set.
*/
- public boolean isOutlinksDirty(java.util.Map value) {
- return isDirty(20);
+ public boolean isOutlinksDirty(java.util.Map value) {
+ return isDirty(19);
}
/**
* Gets the value of the 'inlinks' field.
- */
- public java.util.Map getInlinks() {
+ * Embedded hyperlinks which link to pages within the current domain. */
+ public java.util.Map getInlinks() {
return inlinks;
}
/**
* Sets the value of the 'inlinks' field.
- * @param value the value to set.
+ * Embedded hyperlinks which link to pages within the current domain. * @param value the value to set.
*/
- public void setInlinks(java.util.Map value) {
+ public void setInlinks(java.util.Map value) {
this.inlinks = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(21);
+ setDirty(20);
}
/**
* Checks the dirty status of the 'inlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Embedded hyperlinks which link to pages within the current domain. * @param value the value to set.
*/
- public boolean isInlinksDirty(java.util.Map value) {
- return isDirty(21);
+ public boolean isInlinksDirty(java.util.Map value) {
+ return isDirty(20);
}
/**
* Gets the value of the 'markers' field.
- */
- public java.util.Map getMarkers() {
+ * Markers flags which represent user and machine decisions which have affected influenced a WebPage's current state. Markers can be system specific and user machine driven in nature. They are assigned to a WebPage on a job-by-job basis and thier values indicative of what actions should be associated with a WebPage. */
+ public java.util.Map getMarkers() {
return markers;
}
/**
* Sets the value of the 'markers' field.
- * @param value the value to set.
+ * Markers flags which represent user and machine decisions which have affected influenced a WebPage's current state. Markers can be system specific and user machine driven in nature. They are assigned to a WebPage on a job-by-job basis and thier values indicative of what actions should be associated with a WebPage. * @param value the value to set.
*/
- public void setMarkers(java.util.Map value) {
+ public void setMarkers(java.util.Map value) {
this.markers = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(22);
+ setDirty(21);
}
/**
* Checks the dirty status of the 'markers' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Markers flags which represent user and machine decisions which have affected influenced a WebPage's current state. Markers can be system specific and user machine driven in nature. They are assigned to a WebPage on a job-by-job basis and thier values indicative of what actions should be associated with a WebPage. * @param value the value to set.
*/
- public boolean isMarkersDirty(java.util.Map value) {
- return isDirty(22);
+ public boolean isMarkersDirty(java.util.Map value) {
+ return isDirty(21);
}
/**
* Gets the value of the 'metadata' field.
- */
- public java.util.Map getMetadata() {
+ * A multi-valued metadata container used for storing everything from structured WebPage characterists, to ad-hoc extraction and metadata augmentation for any given WebPage. */
+ public java.util.Map getMetadata() {
return metadata;
}
/**
* Sets the value of the 'metadata' field.
- * @param value the value to set.
+ * A multi-valued metadata container used for storing everything from structured WebPage characterists, to ad-hoc extraction and metadata augmentation for any given WebPage. * @param value the value to set.
*/
- public void setMetadata(java.util.Map value) {
+ public void setMetadata(java.util.Map value) {
this.metadata = (value instanceof org.apache.gora.persistency.Dirtyable) ? value : new org.apache.gora.persistency.impl.DirtyMapWrapper(value);
- setDirty(23);
+ setDirty(22);
}
/**
* Checks the dirty status of the 'metadata' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A multi-valued metadata container used for storing everything from structured WebPage characterists, to ad-hoc extraction and metadata augmentation for any given WebPage. * @param value the value to set.
*/
- public boolean isMetadataDirty(java.util.Map value) {
- return isDirty(23);
+ public boolean isMetadataDirty(java.util.Map value) {
+ return isDirty(22);
}
/**
* Gets the value of the 'batchId' field.
- */
- public CharSequence getBatchId() {
+ * A batchId that this WebPage is assigned to. WebPage's are fetched in batches, called fetchlists. Pages are partitioned but can always be associated and fetched alongside pages of similar value (within a crawl cycle) based on batchId. */
+ public java.lang.CharSequence getBatchId() {
return batchId;
}
/**
* Sets the value of the 'batchId' field.
- * @param value the value to set.
+ * A batchId that this WebPage is assigned to. WebPage's are fetched in batches, called fetchlists. Pages are partitioned but can always be associated and fetched alongside pages of similar value (within a crawl cycle) based on batchId. * @param value the value to set.
*/
- public void setBatchId(CharSequence value) {
+ public void setBatchId(java.lang.CharSequence value) {
this.batchId = value;
- setDirty(24);
+ setDirty(23);
}
/**
* Checks the dirty status of the 'batchId' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A batchId that this WebPage is assigned to. WebPage's are fetched in batches, called fetchlists. Pages are partitioned but can always be associated and fetched alongside pages of similar value (within a crawl cycle) based on batchId. * @param value the value to set.
*/
- public boolean isBatchIdDirty(CharSequence value) {
- return isDirty(24);
+ public boolean isBatchIdDirty(java.lang.CharSequence value) {
+ return isDirty(23);
}
/** Creates a new WebPage RecordBuilder */
- public static Builder newBuilder() {
- return new Builder();
+ public static org.apache.nutch.storage.WebPage.Builder newBuilder() {
+ return new org.apache.nutch.storage.WebPage.Builder();
}
/** Creates a new WebPage RecordBuilder by copying an existing Builder */
- public static Builder newBuilder(Builder other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.WebPage.Builder newBuilder(org.apache.nutch.storage.WebPage.Builder other) {
+ return new org.apache.nutch.storage.WebPage.Builder(other);
}
/** Creates a new WebPage RecordBuilder by copying an existing WebPage instance */
- public static Builder newBuilder(WebPage other) {
- return new Builder(other);
+ public static org.apache.nutch.storage.WebPage.Builder newBuilder(org.apache.nutch.storage.WebPage other) {
+ return new org.apache.nutch.storage.WebPage.Builder(other);
}
- private static java.nio.ByteBuffer deepCopyToWriteOnlyBuffer(
+ private static java.nio.ByteBuffer deepCopyToReadOnlyBuffer(
java.nio.ByteBuffer input) {
java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity());
int position = input.position();
@@ -837,8 +853,7 @@
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase
implements org.apache.avro.data.RecordBuilder {
- private java.nio.ByteBuffer __g__dirty;
- private CharSequence baseUrl;
+ private java.lang.CharSequence baseUrl;
private int status;
private long fetchTime;
private long prevFetchTime;
@@ -846,353 +861,349 @@
private int retriesSinceFetch;
private long modifiedTime;
private long prevModifiedTime;
- private ProtocolStatus protocolStatus;
+ private org.apache.nutch.storage.ProtocolStatus protocolStatus;
private java.nio.ByteBuffer content;
- private CharSequence contentType;
+ private java.lang.CharSequence contentType;
private java.nio.ByteBuffer prevSignature;
private java.nio.ByteBuffer signature;
- private CharSequence title;
- private CharSequence text;
- private ParseStatus parseStatus;
+ private java.lang.CharSequence title;
+ private java.lang.CharSequence text;
+ private org.apache.nutch.storage.ParseStatus parseStatus;
private float score;
- private CharSequence reprUrl;
- private java.util.Map headers;
- private java.util.Map outlinks;
- private java.util.Map inlinks;
- private java.util.Map markers;
- private java.util.Map metadata;
- private CharSequence batchId;
+ private java.lang.CharSequence reprUrl;
+ private java.util.Map headers;
+ private java.util.Map outlinks;
+ private java.util.Map inlinks;
+ private java.util.Map markers;
+ private java.util.Map metadata;
+ private java.lang.CharSequence batchId;
/** Creates a new Builder */
private Builder() {
- super(WebPage.SCHEMA$);
+ super(org.apache.nutch.storage.WebPage.SCHEMA$);
}
/** Creates a Builder by copying an existing Builder */
- private Builder(Builder other) {
+ private Builder(org.apache.nutch.storage.WebPage.Builder other) {
super(other);
}
/** Creates a Builder by copying an existing WebPage instance */
- private Builder(WebPage other) {
- super(WebPage.SCHEMA$);
- if (isValidValue(fields()[0], other.__g__dirty)) {
- this.__g__dirty = (java.nio.ByteBuffer) data().deepCopy(fields()[0].schema(), other.__g__dirty);
+ private Builder(org.apache.nutch.storage.WebPage other) {
+ super(org.apache.nutch.storage.WebPage.SCHEMA$);
+ if (isValidValue(fields()[0], other.baseUrl)) {
+ this.baseUrl = (java.lang.CharSequence) data().deepCopy(fields()[0].schema(), other.baseUrl);
fieldSetFlags()[0] = true;
}
- if (isValidValue(fields()[1], other.baseUrl)) {
- this.baseUrl = (CharSequence) data().deepCopy(fields()[1].schema(), other.baseUrl);
+ if (isValidValue(fields()[1], other.status)) {
+ this.status = (java.lang.Integer) data().deepCopy(fields()[1].schema(), other.status);
fieldSetFlags()[1] = true;
}
- if (isValidValue(fields()[2], other.status)) {
- this.status = (Integer) data().deepCopy(fields()[2].schema(), other.status);
+ if (isValidValue(fields()[2], other.fetchTime)) {
+ this.fetchTime = (java.lang.Long) data().deepCopy(fields()[2].schema(), other.fetchTime);
fieldSetFlags()[2] = true;
}
- if (isValidValue(fields()[3], other.fetchTime)) {
- this.fetchTime = (Long) data().deepCopy(fields()[3].schema(), other.fetchTime);
+ if (isValidValue(fields()[3], other.prevFetchTime)) {
+ this.prevFetchTime = (java.lang.Long) data().deepCopy(fields()[3].schema(), other.prevFetchTime);
fieldSetFlags()[3] = true;
}
- if (isValidValue(fields()[4], other.prevFetchTime)) {
- this.prevFetchTime = (Long) data().deepCopy(fields()[4].schema(), other.prevFetchTime);
+ if (isValidValue(fields()[4], other.fetchInterval)) {
+ this.fetchInterval = (java.lang.Integer) data().deepCopy(fields()[4].schema(), other.fetchInterval);
fieldSetFlags()[4] = true;
}
- if (isValidValue(fields()[5], other.fetchInterval)) {
- this.fetchInterval = (Integer) data().deepCopy(fields()[5].schema(), other.fetchInterval);
+ if (isValidValue(fields()[5], other.retriesSinceFetch)) {
+ this.retriesSinceFetch = (java.lang.Integer) data().deepCopy(fields()[5].schema(), other.retriesSinceFetch);
fieldSetFlags()[5] = true;
}
- if (isValidValue(fields()[6], other.retriesSinceFetch)) {
- this.retriesSinceFetch = (Integer) data().deepCopy(fields()[6].schema(), other.retriesSinceFetch);
+ if (isValidValue(fields()[6], other.modifiedTime)) {
+ this.modifiedTime = (java.lang.Long) data().deepCopy(fields()[6].schema(), other.modifiedTime);
fieldSetFlags()[6] = true;
}
- if (isValidValue(fields()[7], other.modifiedTime)) {
- this.modifiedTime = (Long) data().deepCopy(fields()[7].schema(), other.modifiedTime);
+ if (isValidValue(fields()[7], other.prevModifiedTime)) {
+ this.prevModifiedTime = (java.lang.Long) data().deepCopy(fields()[7].schema(), other.prevModifiedTime);
fieldSetFlags()[7] = true;
}
- if (isValidValue(fields()[8], other.prevModifiedTime)) {
- this.prevModifiedTime = (Long) data().deepCopy(fields()[8].schema(), other.prevModifiedTime);
+ if (isValidValue(fields()[8], other.protocolStatus)) {
+ this.protocolStatus = (org.apache.nutch.storage.ProtocolStatus) data().deepCopy(fields()[8].schema(), other.protocolStatus);
fieldSetFlags()[8] = true;
}
- if (isValidValue(fields()[9], other.protocolStatus)) {
- this.protocolStatus = (ProtocolStatus) data().deepCopy(fields()[9].schema(), other.protocolStatus);
+ if (isValidValue(fields()[9], other.content)) {
+ this.content = (java.nio.ByteBuffer) data().deepCopy(fields()[9].schema(), other.content);
fieldSetFlags()[9] = true;
}
- if (isValidValue(fields()[10], other.content)) {
- this.content = (java.nio.ByteBuffer) data().deepCopy(fields()[10].schema(), other.content);
+ if (isValidValue(fields()[10], other.contentType)) {
+ this.contentType = (java.lang.CharSequence) data().deepCopy(fields()[10].schema(), other.contentType);
fieldSetFlags()[10] = true;
}
- if (isValidValue(fields()[11], other.contentType)) {
- this.contentType = (CharSequence) data().deepCopy(fields()[11].schema(), other.contentType);
+ if (isValidValue(fields()[11], other.prevSignature)) {
+ this.prevSignature = (java.nio.ByteBuffer) data().deepCopy(fields()[11].schema(), other.prevSignature);
fieldSetFlags()[11] = true;
}
- if (isValidValue(fields()[12], other.prevSignature)) {
- this.prevSignature = (java.nio.ByteBuffer) data().deepCopy(fields()[12].schema(), other.prevSignature);
+ if (isValidValue(fields()[12], other.signature)) {
+ this.signature = (java.nio.ByteBuffer) data().deepCopy(fields()[12].schema(), other.signature);
fieldSetFlags()[12] = true;
}
- if (isValidValue(fields()[13], other.signature)) {
- this.signature = (java.nio.ByteBuffer) data().deepCopy(fields()[13].schema(), other.signature);
+ if (isValidValue(fields()[13], other.title)) {
+ this.title = (java.lang.CharSequence) data().deepCopy(fields()[13].schema(), other.title);
fieldSetFlags()[13] = true;
}
- if (isValidValue(fields()[14], other.title)) {
- this.title = (CharSequence) data().deepCopy(fields()[14].schema(), other.title);
+ if (isValidValue(fields()[14], other.text)) {
+ this.text = (java.lang.CharSequence) data().deepCopy(fields()[14].schema(), other.text);
fieldSetFlags()[14] = true;
}
- if (isValidValue(fields()[15], other.text)) {
- this.text = (CharSequence) data().deepCopy(fields()[15].schema(), other.text);
+ if (isValidValue(fields()[15], other.parseStatus)) {
+ this.parseStatus = (org.apache.nutch.storage.ParseStatus) data().deepCopy(fields()[15].schema(), other.parseStatus);
fieldSetFlags()[15] = true;
}
- if (isValidValue(fields()[16], other.parseStatus)) {
- this.parseStatus = (ParseStatus) data().deepCopy(fields()[16].schema(), other.parseStatus);
+ if (isValidValue(fields()[16], other.score)) {
+ this.score = (java.lang.Float) data().deepCopy(fields()[16].schema(), other.score);
fieldSetFlags()[16] = true;
}
- if (isValidValue(fields()[17], other.score)) {
- this.score = (Float) data().deepCopy(fields()[17].schema(), other.score);
+ if (isValidValue(fields()[17], other.reprUrl)) {
+ this.reprUrl = (java.lang.CharSequence) data().deepCopy(fields()[17].schema(), other.reprUrl);
fieldSetFlags()[17] = true;
}
- if (isValidValue(fields()[18], other.reprUrl)) {
- this.reprUrl = (CharSequence) data().deepCopy(fields()[18].schema(), other.reprUrl);
+ if (isValidValue(fields()[18], other.headers)) {
+ this.headers = (java.util.Map) data().deepCopy(fields()[18].schema(), other.headers);
fieldSetFlags()[18] = true;
}
- if (isValidValue(fields()[19], other.headers)) {
- this.headers = (java.util.Map) data().deepCopy(fields()[19].schema(), other.headers);
+ if (isValidValue(fields()[19], other.outlinks)) {
+ this.outlinks = (java.util.Map) data().deepCopy(fields()[19].schema(), other.outlinks);
fieldSetFlags()[19] = true;
}
- if (isValidValue(fields()[20], other.outlinks)) {
- this.outlinks = (java.util.Map) data().deepCopy(fields()[20].schema(), other.outlinks);
+ if (isValidValue(fields()[20], other.inlinks)) {
+ this.inlinks = (java.util.Map) data().deepCopy(fields()[20].schema(), other.inlinks);
fieldSetFlags()[20] = true;
}
- if (isValidValue(fields()[21], other.inlinks)) {
- this.inlinks = (java.util.Map) data().deepCopy(fields()[21].schema(), other.inlinks);
+ if (isValidValue(fields()[21], other.markers)) {
+ this.markers = (java.util.Map) data().deepCopy(fields()[21].schema(), other.markers);
fieldSetFlags()[21] = true;
}
- if (isValidValue(fields()[22], other.markers)) {
- this.markers = (java.util.Map) data().deepCopy(fields()[22].schema(), other.markers);
+ if (isValidValue(fields()[22], other.metadata)) {
+ this.metadata = (java.util.Map) data().deepCopy(fields()[22].schema(), other.metadata);
fieldSetFlags()[22] = true;
}
- if (isValidValue(fields()[23], other.metadata)) {
- this.metadata = (java.util.Map) data().deepCopy(fields()[23].schema(), other.metadata);
+ if (isValidValue(fields()[23], other.batchId)) {
+ this.batchId = (java.lang.CharSequence) data().deepCopy(fields()[23].schema(), other.batchId);
fieldSetFlags()[23] = true;
}
- if (isValidValue(fields()[24], other.batchId)) {
- this.batchId = (CharSequence) data().deepCopy(fields()[24].schema(), other.batchId);
- fieldSetFlags()[24] = true;
- }
}
/** Gets the value of the 'baseUrl' field */
- public CharSequence getBaseUrl() {
+ public java.lang.CharSequence getBaseUrl() {
return baseUrl;
}
/** Sets the value of the 'baseUrl' field */
- public Builder setBaseUrl(CharSequence value) {
- validate(fields()[1], value);
+ public org.apache.nutch.storage.WebPage.Builder setBaseUrl(java.lang.CharSequence value) {
+ validate(fields()[0], value);
this.baseUrl = value;
- fieldSetFlags()[1] = true;
+ fieldSetFlags()[0] = true;
return this;
}
/** Checks whether the 'baseUrl' field has been set */
public boolean hasBaseUrl() {
- return fieldSetFlags()[1];
+ return fieldSetFlags()[0];
}
/** Clears the value of the 'baseUrl' field */
- public Builder clearBaseUrl() {
+ public org.apache.nutch.storage.WebPage.Builder clearBaseUrl() {
baseUrl = null;
- fieldSetFlags()[1] = false;
+ fieldSetFlags()[0] = false;
return this;
}
/** Gets the value of the 'status' field */
- public Integer getStatus() {
+ public java.lang.Integer getStatus() {
return status;
}
/** Sets the value of the 'status' field */
- public Builder setStatus(int value) {
- validate(fields()[2], value);
+ public org.apache.nutch.storage.WebPage.Builder setStatus(int value) {
+ validate(fields()[1], value);
this.status = value;
- fieldSetFlags()[2] = true;
+ fieldSetFlags()[1] = true;
return this;
}
/** Checks whether the 'status' field has been set */
public boolean hasStatus() {
- return fieldSetFlags()[2];
+ return fieldSetFlags()[1];
}
/** Clears the value of the 'status' field */
- public Builder clearStatus() {
- fieldSetFlags()[2] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearStatus() {
+ fieldSetFlags()[1] = false;
return this;
}
/** Gets the value of the 'fetchTime' field */
- public Long getFetchTime() {
+ public java.lang.Long getFetchTime() {
return fetchTime;
}
/** Sets the value of the 'fetchTime' field */
- public Builder setFetchTime(long value) {
- validate(fields()[3], value);
+ public org.apache.nutch.storage.WebPage.Builder setFetchTime(long value) {
+ validate(fields()[2], value);
this.fetchTime = value;
- fieldSetFlags()[3] = true;
+ fieldSetFlags()[2] = true;
return this;
}
/** Checks whether the 'fetchTime' field has been set */
public boolean hasFetchTime() {
- return fieldSetFlags()[3];
+ return fieldSetFlags()[2];
}
/** Clears the value of the 'fetchTime' field */
- public Builder clearFetchTime() {
- fieldSetFlags()[3] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearFetchTime() {
+ fieldSetFlags()[2] = false;
return this;
}
/** Gets the value of the 'prevFetchTime' field */
- public Long getPrevFetchTime() {
+ public java.lang.Long getPrevFetchTime() {
return prevFetchTime;
}
/** Sets the value of the 'prevFetchTime' field */
- public Builder setPrevFetchTime(long value) {
- validate(fields()[4], value);
+ public org.apache.nutch.storage.WebPage.Builder setPrevFetchTime(long value) {
+ validate(fields()[3], value);
this.prevFetchTime = value;
- fieldSetFlags()[4] = true;
+ fieldSetFlags()[3] = true;
return this;
}
/** Checks whether the 'prevFetchTime' field has been set */
public boolean hasPrevFetchTime() {
- return fieldSetFlags()[4];
+ return fieldSetFlags()[3];
}
/** Clears the value of the 'prevFetchTime' field */
- public Builder clearPrevFetchTime() {
- fieldSetFlags()[4] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearPrevFetchTime() {
+ fieldSetFlags()[3] = false;
return this;
}
/** Gets the value of the 'fetchInterval' field */
- public Integer getFetchInterval() {
+ public java.lang.Integer getFetchInterval() {
return fetchInterval;
}
/** Sets the value of the 'fetchInterval' field */
- public Builder setFetchInterval(int value) {
- validate(fields()[5], value);
+ public org.apache.nutch.storage.WebPage.Builder setFetchInterval(int value) {
+ validate(fields()[4], value);
this.fetchInterval = value;
- fieldSetFlags()[5] = true;
+ fieldSetFlags()[4] = true;
return this;
}
/** Checks whether the 'fetchInterval' field has been set */
public boolean hasFetchInterval() {
- return fieldSetFlags()[5];
+ return fieldSetFlags()[4];
}
/** Clears the value of the 'fetchInterval' field */
- public Builder clearFetchInterval() {
- fieldSetFlags()[5] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearFetchInterval() {
+ fieldSetFlags()[4] = false;
return this;
}
/** Gets the value of the 'retriesSinceFetch' field */
- public Integer getRetriesSinceFetch() {
+ public java.lang.Integer getRetriesSinceFetch() {
return retriesSinceFetch;
}
/** Sets the value of the 'retriesSinceFetch' field */
- public Builder setRetriesSinceFetch(int value) {
- validate(fields()[6], value);
+ public org.apache.nutch.storage.WebPage.Builder setRetriesSinceFetch(int value) {
+ validate(fields()[5], value);
this.retriesSinceFetch = value;
- fieldSetFlags()[6] = true;
+ fieldSetFlags()[5] = true;
return this;
}
/** Checks whether the 'retriesSinceFetch' field has been set */
public boolean hasRetriesSinceFetch() {
- return fieldSetFlags()[6];
+ return fieldSetFlags()[5];
}
/** Clears the value of the 'retriesSinceFetch' field */
- public Builder clearRetriesSinceFetch() {
- fieldSetFlags()[6] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearRetriesSinceFetch() {
+ fieldSetFlags()[5] = false;
return this;
}
/** Gets the value of the 'modifiedTime' field */
- public Long getModifiedTime() {
+ public java.lang.Long getModifiedTime() {
return modifiedTime;
}
/** Sets the value of the 'modifiedTime' field */
- public Builder setModifiedTime(long value) {
- validate(fields()[7], value);
+ public org.apache.nutch.storage.WebPage.Builder setModifiedTime(long value) {
+ validate(fields()[6], value);
this.modifiedTime = value;
- fieldSetFlags()[7] = true;
+ fieldSetFlags()[6] = true;
return this;
}
/** Checks whether the 'modifiedTime' field has been set */
public boolean hasModifiedTime() {
- return fieldSetFlags()[7];
+ return fieldSetFlags()[6];
}
/** Clears the value of the 'modifiedTime' field */
- public Builder clearModifiedTime() {
- fieldSetFlags()[7] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearModifiedTime() {
+ fieldSetFlags()[6] = false;
return this;
}
/** Gets the value of the 'prevModifiedTime' field */
- public Long getPrevModifiedTime() {
+ public java.lang.Long getPrevModifiedTime() {
return prevModifiedTime;
}
/** Sets the value of the 'prevModifiedTime' field */
- public Builder setPrevModifiedTime(long value) {
- validate(fields()[8], value);
+ public org.apache.nutch.storage.WebPage.Builder setPrevModifiedTime(long value) {
+ validate(fields()[7], value);
this.prevModifiedTime = value;
- fieldSetFlags()[8] = true;
+ fieldSetFlags()[7] = true;
return this;
}
/** Checks whether the 'prevModifiedTime' field has been set */
public boolean hasPrevModifiedTime() {
- return fieldSetFlags()[8];
+ return fieldSetFlags()[7];
}
/** Clears the value of the 'prevModifiedTime' field */
- public Builder clearPrevModifiedTime() {
- fieldSetFlags()[8] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearPrevModifiedTime() {
+ fieldSetFlags()[7] = false;
return this;
}
/** Gets the value of the 'protocolStatus' field */
- public ProtocolStatus getProtocolStatus() {
+ public org.apache.nutch.storage.ProtocolStatus getProtocolStatus() {
return protocolStatus;
}
/** Sets the value of the 'protocolStatus' field */
- public Builder setProtocolStatus(ProtocolStatus value) {
- validate(fields()[9], value);
+ public org.apache.nutch.storage.WebPage.Builder setProtocolStatus(org.apache.nutch.storage.ProtocolStatus value) {
+ validate(fields()[8], value);
this.protocolStatus = value;
- fieldSetFlags()[9] = true;
+ fieldSetFlags()[8] = true;
return this;
}
/** Checks whether the 'protocolStatus' field has been set */
public boolean hasProtocolStatus() {
- return fieldSetFlags()[9];
+ return fieldSetFlags()[8];
}
/** Clears the value of the 'protocolStatus' field */
- public Builder clearProtocolStatus() {
+ public org.apache.nutch.storage.WebPage.Builder clearProtocolStatus() {
protocolStatus = null;
- fieldSetFlags()[9] = false;
+ fieldSetFlags()[8] = false;
return this;
}
@@ -1202,47 +1213,47 @@
}
/** Sets the value of the 'content' field */
- public Builder setContent(java.nio.ByteBuffer value) {
- validate(fields()[10], value);
+ public org.apache.nutch.storage.WebPage.Builder setContent(java.nio.ByteBuffer value) {
+ validate(fields()[9], value);
this.content = value;
- fieldSetFlags()[10] = true;
+ fieldSetFlags()[9] = true;
return this;
}
/** Checks whether the 'content' field has been set */
public boolean hasContent() {
- return fieldSetFlags()[10];
+ return fieldSetFlags()[9];
}
/** Clears the value of the 'content' field */
- public Builder clearContent() {
+ public org.apache.nutch.storage.WebPage.Builder clearContent() {
content = null;
- fieldSetFlags()[10] = false;
+ fieldSetFlags()[9] = false;
return this;
}
/** Gets the value of the 'contentType' field */
- public CharSequence getContentType() {
+ public java.lang.CharSequence getContentType() {
return contentType;
}
/** Sets the value of the 'contentType' field */
- public Builder setContentType(CharSequence value) {
- validate(fields()[11], value);
+ public org.apache.nutch.storage.WebPage.Builder setContentType(java.lang.CharSequence value) {
+ validate(fields()[10], value);
this.contentType = value;
- fieldSetFlags()[11] = true;
+ fieldSetFlags()[10] = true;
return this;
}
/** Checks whether the 'contentType' field has been set */
public boolean hasContentType() {
- return fieldSetFlags()[11];
+ return fieldSetFlags()[10];
}
/** Clears the value of the 'contentType' field */
- public Builder clearContentType() {
+ public org.apache.nutch.storage.WebPage.Builder clearContentType() {
contentType = null;
- fieldSetFlags()[11] = false;
+ fieldSetFlags()[10] = false;
return this;
}
@@ -1252,22 +1263,22 @@
}
/** Sets the value of the 'prevSignature' field */
- public Builder setPrevSignature(java.nio.ByteBuffer value) {
- validate(fields()[12], value);
+ public org.apache.nutch.storage.WebPage.Builder setPrevSignature(java.nio.ByteBuffer value) {
+ validate(fields()[11], value);
this.prevSignature = value;
- fieldSetFlags()[12] = true;
+ fieldSetFlags()[11] = true;
return this;
}
/** Checks whether the 'prevSignature' field has been set */
public boolean hasPrevSignature() {
- return fieldSetFlags()[12];
+ return fieldSetFlags()[11];
}
/** Clears the value of the 'prevSignature' field */
- public Builder clearPrevSignature() {
+ public org.apache.nutch.storage.WebPage.Builder clearPrevSignature() {
prevSignature = null;
- fieldSetFlags()[12] = false;
+ fieldSetFlags()[11] = false;
return this;
}
@@ -1277,296 +1288,296 @@
}
/** Sets the value of the 'signature' field */
- public Builder setSignature(java.nio.ByteBuffer value) {
- validate(fields()[13], value);
+ public org.apache.nutch.storage.WebPage.Builder setSignature(java.nio.ByteBuffer value) {
+ validate(fields()[12], value);
this.signature = value;
- fieldSetFlags()[13] = true;
+ fieldSetFlags()[12] = true;
return this;
}
/** Checks whether the 'signature' field has been set */
public boolean hasSignature() {
- return fieldSetFlags()[13];
+ return fieldSetFlags()[12];
}
/** Clears the value of the 'signature' field */
- public Builder clearSignature() {
+ public org.apache.nutch.storage.WebPage.Builder clearSignature() {
signature = null;
- fieldSetFlags()[13] = false;
+ fieldSetFlags()[12] = false;
return this;
}
/** Gets the value of the 'title' field */
- public CharSequence getTitle() {
+ public java.lang.CharSequence getTitle() {
return title;
}
/** Sets the value of the 'title' field */
- public Builder setTitle(CharSequence value) {
- validate(fields()[14], value);
+ public org.apache.nutch.storage.WebPage.Builder setTitle(java.lang.CharSequence value) {
+ validate(fields()[13], value);
this.title = value;
- fieldSetFlags()[14] = true;
+ fieldSetFlags()[13] = true;
return this;
}
/** Checks whether the 'title' field has been set */
public boolean hasTitle() {
- return fieldSetFlags()[14];
+ return fieldSetFlags()[13];
}
/** Clears the value of the 'title' field */
- public Builder clearTitle() {
+ public org.apache.nutch.storage.WebPage.Builder clearTitle() {
title = null;
- fieldSetFlags()[14] = false;
+ fieldSetFlags()[13] = false;
return this;
}
/** Gets the value of the 'text' field */
- public CharSequence getText() {
+ public java.lang.CharSequence getText() {
return text;
}
/** Sets the value of the 'text' field */
- public Builder setText(CharSequence value) {
- validate(fields()[15], value);
+ public org.apache.nutch.storage.WebPage.Builder setText(java.lang.CharSequence value) {
+ validate(fields()[14], value);
this.text = value;
- fieldSetFlags()[15] = true;
+ fieldSetFlags()[14] = true;
return this;
}
/** Checks whether the 'text' field has been set */
public boolean hasText() {
- return fieldSetFlags()[15];
+ return fieldSetFlags()[14];
}
/** Clears the value of the 'text' field */
- public Builder clearText() {
+ public org.apache.nutch.storage.WebPage.Builder clearText() {
text = null;
- fieldSetFlags()[15] = false;
+ fieldSetFlags()[14] = false;
return this;
}
/** Gets the value of the 'parseStatus' field */
- public ParseStatus getParseStatus() {
+ public org.apache.nutch.storage.ParseStatus getParseStatus() {
return parseStatus;
}
/** Sets the value of the 'parseStatus' field */
- public Builder setParseStatus(ParseStatus value) {
- validate(fields()[16], value);
+ public org.apache.nutch.storage.WebPage.Builder setParseStatus(org.apache.nutch.storage.ParseStatus value) {
+ validate(fields()[15], value);
this.parseStatus = value;
- fieldSetFlags()[16] = true;
+ fieldSetFlags()[15] = true;
return this;
}
/** Checks whether the 'parseStatus' field has been set */
public boolean hasParseStatus() {
- return fieldSetFlags()[16];
+ return fieldSetFlags()[15];
}
/** Clears the value of the 'parseStatus' field */
- public Builder clearParseStatus() {
+ public org.apache.nutch.storage.WebPage.Builder clearParseStatus() {
parseStatus = null;
- fieldSetFlags()[16] = false;
+ fieldSetFlags()[15] = false;
return this;
}
/** Gets the value of the 'score' field */
- public Float getScore() {
+ public java.lang.Float getScore() {
return score;
}
/** Sets the value of the 'score' field */
- public Builder setScore(float value) {
- validate(fields()[17], value);
+ public org.apache.nutch.storage.WebPage.Builder setScore(float value) {
+ validate(fields()[16], value);
this.score = value;
- fieldSetFlags()[17] = true;
+ fieldSetFlags()[16] = true;
return this;
}
/** Checks whether the 'score' field has been set */
public boolean hasScore() {
- return fieldSetFlags()[17];
+ return fieldSetFlags()[16];
}
/** Clears the value of the 'score' field */
- public Builder clearScore() {
- fieldSetFlags()[17] = false;
+ public org.apache.nutch.storage.WebPage.Builder clearScore() {
+ fieldSetFlags()[16] = false;
return this;
}
/** Gets the value of the 'reprUrl' field */
- public CharSequence getReprUrl() {
+ public java.lang.CharSequence getReprUrl() {
return reprUrl;
}
/** Sets the value of the 'reprUrl' field */
- public Builder setReprUrl(CharSequence value) {
- validate(fields()[18], value);
+ public org.apache.nutch.storage.WebPage.Builder setReprUrl(java.lang.CharSequence value) {
+ validate(fields()[17], value);
this.reprUrl = value;
- fieldSetFlags()[18] = true;
+ fieldSetFlags()[17] = true;
return this;
}
/** Checks whether the 'reprUrl' field has been set */
public boolean hasReprUrl() {
- return fieldSetFlags()[18];
+ return fieldSetFlags()[17];
}
/** Clears the value of the 'reprUrl' field */
- public Builder clearReprUrl() {
+ public org.apache.nutch.storage.WebPage.Builder clearReprUrl() {
reprUrl = null;
- fieldSetFlags()[18] = false;
+ fieldSetFlags()[17] = false;
return this;
}
/** Gets the value of the 'headers' field */
- public java.util.Map getHeaders() {
+ public java.util.Map getHeaders() {
return headers;
}
/** Sets the value of the 'headers' field */
- public Builder setHeaders(java.util.Map value) {
- validate(fields()[19], value);
+ public org.apache.nutch.storage.WebPage.Builder setHeaders(java.util.Map value) {
+ validate(fields()[18], value);
this.headers = value;
- fieldSetFlags()[19] = true;
+ fieldSetFlags()[18] = true;
return this;
}
/** Checks whether the 'headers' field has been set */
public boolean hasHeaders() {
- return fieldSetFlags()[19];
+ return fieldSetFlags()[18];
}
/** Clears the value of the 'headers' field */
- public Builder clearHeaders() {
+ public org.apache.nutch.storage.WebPage.Builder clearHeaders() {
headers = null;
- fieldSetFlags()[19] = false;
+ fieldSetFlags()[18] = false;
return this;
}
/** Gets the value of the 'outlinks' field */
- public java.util.Map getOutlinks() {
+ public java.util.Map getOutlinks() {
return outlinks;
}
/** Sets the value of the 'outlinks' field */
- public Builder setOutlinks(java.util.Map value) {
- validate(fields()[20], value);
+ public org.apache.nutch.storage.WebPage.Builder setOutlinks(java.util.Map value) {
+ validate(fields()[19], value);
this.outlinks = value;
- fieldSetFlags()[20] = true;
+ fieldSetFlags()[19] = true;
return this;
}
/** Checks whether the 'outlinks' field has been set */
public boolean hasOutlinks() {
- return fieldSetFlags()[20];
+ return fieldSetFlags()[19];
}
/** Clears the value of the 'outlinks' field */
- public Builder clearOutlinks() {
+ public org.apache.nutch.storage.WebPage.Builder clearOutlinks() {
outlinks = null;
- fieldSetFlags()[20] = false;
+ fieldSetFlags()[19] = false;
return this;
}
/** Gets the value of the 'inlinks' field */
- public java.util.Map getInlinks() {
+ public java.util.Map getInlinks() {
return inlinks;
}
/** Sets the value of the 'inlinks' field */
- public Builder setInlinks(java.util.Map value) {
- validate(fields()[21], value);
+ public org.apache.nutch.storage.WebPage.Builder setInlinks(java.util.Map value) {
+ validate(fields()[20], value);
this.inlinks = value;
- fieldSetFlags()[21] = true;
+ fieldSetFlags()[20] = true;
return this;
}
/** Checks whether the 'inlinks' field has been set */
public boolean hasInlinks() {
- return fieldSetFlags()[21];
+ return fieldSetFlags()[20];
}
/** Clears the value of the 'inlinks' field */
- public Builder clearInlinks() {
+ public org.apache.nutch.storage.WebPage.Builder clearInlinks() {
inlinks = null;
- fieldSetFlags()[21] = false;
+ fieldSetFlags()[20] = false;
return this;
}
/** Gets the value of the 'markers' field */
- public java.util.Map getMarkers() {
+ public java.util.Map getMarkers() {
return markers;
}
/** Sets the value of the 'markers' field */
- public Builder setMarkers(java.util.Map value) {
- validate(fields()[22], value);
+ public org.apache.nutch.storage.WebPage.Builder setMarkers(java.util.Map value) {
+ validate(fields()[21], value);
this.markers = value;
- fieldSetFlags()[22] = true;
+ fieldSetFlags()[21] = true;
return this;
}
/** Checks whether the 'markers' field has been set */
public boolean hasMarkers() {
- return fieldSetFlags()[22];
+ return fieldSetFlags()[21];
}
/** Clears the value of the 'markers' field */
- public Builder clearMarkers() {
+ public org.apache.nutch.storage.WebPage.Builder clearMarkers() {
markers = null;
- fieldSetFlags()[22] = false;
+ fieldSetFlags()[21] = false;
return this;
}
/** Gets the value of the 'metadata' field */
- public java.util.Map getMetadata() {
+ public java.util.Map getMetadata() {
return metadata;
}
/** Sets the value of the 'metadata' field */
- public Builder setMetadata(java.util.Map value) {
- validate(fields()[23], value);
+ public org.apache.nutch.storage.WebPage.Builder setMetadata(java.util.Map value) {
+ validate(fields()[22], value);
this.metadata = value;
- fieldSetFlags()[23] = true;
+ fieldSetFlags()[22] = true;
return this;
}
/** Checks whether the 'metadata' field has been set */
public boolean hasMetadata() {
- return fieldSetFlags()[23];
+ return fieldSetFlags()[22];
}
/** Clears the value of the 'metadata' field */
- public Builder clearMetadata() {
+ public org.apache.nutch.storage.WebPage.Builder clearMetadata() {
metadata = null;
- fieldSetFlags()[23] = false;
+ fieldSetFlags()[22] = false;
return this;
}
/** Gets the value of the 'batchId' field */
- public CharSequence getBatchId() {
+ public java.lang.CharSequence getBatchId() {
return batchId;
}
/** Sets the value of the 'batchId' field */
- public Builder setBatchId(CharSequence value) {
- validate(fields()[24], value);
+ public org.apache.nutch.storage.WebPage.Builder setBatchId(java.lang.CharSequence value) {
+ validate(fields()[23], value);
this.batchId = value;
- fieldSetFlags()[24] = true;
+ fieldSetFlags()[23] = true;
return this;
}
/** Checks whether the 'batchId' field has been set */
public boolean hasBatchId() {
- return fieldSetFlags()[24];
+ return fieldSetFlags()[23];
}
/** Clears the value of the 'batchId' field */
- public Builder clearBatchId() {
+ public org.apache.nutch.storage.WebPage.Builder clearBatchId() {
batchId = null;
- fieldSetFlags()[24] = false;
+ fieldSetFlags()[23] = false;
return this;
}
@@ -1574,31 +1585,30 @@
public WebPage build() {
try {
WebPage record = new WebPage();
- record.__g__dirty = fieldSetFlags()[0] ? this.__g__dirty : (java.nio.ByteBuffer) java.nio.ByteBuffer.wrap(new byte[4]);
- record.baseUrl = fieldSetFlags()[1] ? this.baseUrl : (CharSequence) defaultValue(fields()[1]);
- record.status = fieldSetFlags()[2] ? this.status : (Integer) defaultValue(fields()[2]);
- record.fetchTime = fieldSetFlags()[3] ? this.fetchTime : (Long) defaultValue(fields()[3]);
- record.prevFetchTime = fieldSetFlags()[4] ? this.prevFetchTime : (Long) defaultValue(fields()[4]);
- record.fetchInterval = fieldSetFlags()[5] ? this.fetchInterval : (Integer) defaultValue(fields()[5]);
- record.retriesSinceFetch = fieldSetFlags()[6] ? this.retriesSinceFetch : (Integer) defaultValue(fields()[6]);
- record.modifiedTime = fieldSetFlags()[7] ? this.modifiedTime : (Long) defaultValue(fields()[7]);
- record.prevModifiedTime = fieldSetFlags()[8] ? this.prevModifiedTime : (Long) defaultValue(fields()[8]);
- record.protocolStatus = fieldSetFlags()[9] ? this.protocolStatus : (ProtocolStatus) defaultValue(fields()[9]);
- record.content = fieldSetFlags()[10] ? this.content : (java.nio.ByteBuffer) defaultValue(fields()[10]);
- record.contentType = fieldSetFlags()[11] ? this.contentType : (CharSequence) defaultValue(fields()[11]);
- record.prevSignature = fieldSetFlags()[12] ? this.prevSignature : (java.nio.ByteBuffer) defaultValue(fields()[12]);
- record.signature = fieldSetFlags()[13] ? this.signature : (java.nio.ByteBuffer) defaultValue(fields()[13]);
- record.title = fieldSetFlags()[14] ? this.title : (CharSequence) defaultValue(fields()[14]);
- record.text = fieldSetFlags()[15] ? this.text : (CharSequence) defaultValue(fields()[15]);
- record.parseStatus = fieldSetFlags()[16] ? this.parseStatus : (ParseStatus) defaultValue(fields()[16]);
- record.score = fieldSetFlags()[17] ? this.score : (Float) defaultValue(fields()[17]);
- record.reprUrl = fieldSetFlags()[18] ? this.reprUrl : (CharSequence) defaultValue(fields()[18]);
- record.headers = fieldSetFlags()[19] ? this.headers : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[19]));
- record.outlinks = fieldSetFlags()[20] ? this.outlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[20]));
- record.inlinks = fieldSetFlags()[21] ? this.inlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[21]));
- record.markers = fieldSetFlags()[22] ? this.markers : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[22]));
- record.metadata = fieldSetFlags()[23] ? this.metadata : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[23]));
- record.batchId = fieldSetFlags()[24] ? this.batchId : (CharSequence) defaultValue(fields()[24]);
+ record.baseUrl = fieldSetFlags()[0] ? this.baseUrl : (java.lang.CharSequence) defaultValue(fields()[0]);
+ record.status = fieldSetFlags()[1] ? this.status : (java.lang.Integer) defaultValue(fields()[1]);
+ record.fetchTime = fieldSetFlags()[2] ? this.fetchTime : (java.lang.Long) defaultValue(fields()[2]);
+ record.prevFetchTime = fieldSetFlags()[3] ? this.prevFetchTime : (java.lang.Long) defaultValue(fields()[3]);
+ record.fetchInterval = fieldSetFlags()[4] ? this.fetchInterval : (java.lang.Integer) defaultValue(fields()[4]);
+ record.retriesSinceFetch = fieldSetFlags()[5] ? this.retriesSinceFetch : (java.lang.Integer) defaultValue(fields()[5]);
+ record.modifiedTime = fieldSetFlags()[6] ? this.modifiedTime : (java.lang.Long) defaultValue(fields()[6]);
+ record.prevModifiedTime = fieldSetFlags()[7] ? this.prevModifiedTime : (java.lang.Long) defaultValue(fields()[7]);
+ record.protocolStatus = fieldSetFlags()[8] ? this.protocolStatus : (org.apache.nutch.storage.ProtocolStatus) defaultValue(fields()[8]);
+ record.content = fieldSetFlags()[9] ? this.content : (java.nio.ByteBuffer) defaultValue(fields()[9]);
+ record.contentType = fieldSetFlags()[10] ? this.contentType : (java.lang.CharSequence) defaultValue(fields()[10]);
+ record.prevSignature = fieldSetFlags()[11] ? this.prevSignature : (java.nio.ByteBuffer) defaultValue(fields()[11]);
+ record.signature = fieldSetFlags()[12] ? this.signature : (java.nio.ByteBuffer) defaultValue(fields()[12]);
+ record.title = fieldSetFlags()[13] ? this.title : (java.lang.CharSequence) defaultValue(fields()[13]);
+ record.text = fieldSetFlags()[14] ? this.text : (java.lang.CharSequence) defaultValue(fields()[14]);
+ record.parseStatus = fieldSetFlags()[15] ? this.parseStatus : (org.apache.nutch.storage.ParseStatus) defaultValue(fields()[15]);
+ record.score = fieldSetFlags()[16] ? this.score : (java.lang.Float) defaultValue(fields()[16]);
+ record.reprUrl = fieldSetFlags()[17] ? this.reprUrl : (java.lang.CharSequence) defaultValue(fields()[17]);
+ record.headers = fieldSetFlags()[18] ? this.headers : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[18]));
+ record.outlinks = fieldSetFlags()[19] ? this.outlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[19]));
+ record.inlinks = fieldSetFlags()[20] ? this.inlinks : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[20]));
+ record.markers = fieldSetFlags()[21] ? this.markers : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[21]));
+ record.metadata = fieldSetFlags()[22] ? this.metadata : (java.util.Map) new org.apache.gora.persistency.impl.DirtyMapWrapper((java.util.Map)defaultValue(fields()[22]));
+ record.batchId = fieldSetFlags()[23] ? this.batchId : (java.lang.CharSequence) defaultValue(fields()[23]);
return record;
} catch (Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
@@ -1606,7 +1616,7 @@
}
}
- public Tombstone getTombstone(){
+ public WebPage.Tombstone getTombstone(){
return TOMBSTONE;
}
@@ -1620,195 +1630,195 @@
private Tombstone() { }
- /**
+ /**
* Gets the value of the 'baseUrl' field.
- */
- public CharSequence getBaseUrl() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The original associated with this WebPage. */
+ public java.lang.CharSequence getBaseUrl() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'baseUrl' field.
- * @param value the value to set.
+ * The original associated with this WebPage. * @param value the value to set.
*/
- public void setBaseUrl(CharSequence value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setBaseUrl(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'baseUrl' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The original associated with this WebPage. * @param value the value to set.
*/
- public boolean isBaseUrlDirty(CharSequence value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isBaseUrlDirty(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'status' field.
- */
- public Integer getStatus() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified */
+ public java.lang.Integer getStatus() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'status' field.
- * @param value the value to set.
+ * A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified * @param value the value to set.
*/
- public void setStatus(Integer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setStatus(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'status' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A crawl status associated with the WebPage, can be of value STATUS_UNFETCHED - WebPage was not fetched yet, STATUS_FETCHED - WebPage was successfully fetched, STATUS_GONE - WebPage no longer exists, STATUS_REDIR_TEMP - WebPage temporarily redirects to other page, STATUS_REDIR_PERM - WebPage permanently redirects to other page, STATUS_RETRY - Fetching unsuccessful, needs to be retried e.g. transient errors and STATUS_NOTMODIFIED - fetching successful - page is not modified * @param value the value to set.
*/
- public boolean isStatusDirty(Integer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isStatusDirty(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'fetchTime' field.
- */
- public Long getFetchTime() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The system time in milliseconds for when the page was fetched. */
+ public java.lang.Long getFetchTime() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'fetchTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was fetched. * @param value the value to set.
*/
- public void setFetchTime(Long value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setFetchTime(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'fetchTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was fetched. * @param value the value to set.
*/
- public boolean isFetchTimeDirty(Long value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isFetchTimeDirty(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'prevFetchTime' field.
- */
- public Long getPrevFetchTime() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation */
+ public java.lang.Long getPrevFetchTime() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'prevFetchTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation * @param value the value to set.
*/
- public void setPrevFetchTime(Long value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setPrevFetchTime(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'prevFetchTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when the page was last fetched if it was previously fetched which can be used to calculate time delta within a fetching schedule implementation * @param value the value to set.
*/
- public boolean isPrevFetchTimeDirty(Long value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isPrevFetchTimeDirty(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'fetchInterval' field.
- */
- public Integer getFetchInterval() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. */
+ public java.lang.Integer getFetchInterval() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'fetchInterval' field.
- * @param value the value to set.
+ * The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. * @param value the value to set.
*/
- public void setFetchInterval(Integer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setFetchInterval(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'fetchInterval' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The default number of seconds between re-fetches of a page. The default is considered as 30 days unless a custom fetch schedle is implemented. * @param value the value to set.
*/
- public boolean isFetchIntervalDirty(Integer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isFetchIntervalDirty(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'retriesSinceFetch' field.
- */
- public Integer getRetriesSinceFetch() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The number of retried attempts at fetching the WebPage since it was last successfully fetched. */
+ public java.lang.Integer getRetriesSinceFetch() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'retriesSinceFetch' field.
- * @param value the value to set.
+ * The number of retried attempts at fetching the WebPage since it was last successfully fetched. * @param value the value to set.
*/
- public void setRetriesSinceFetch(Integer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setRetriesSinceFetch(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'retriesSinceFetch' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The number of retried attempts at fetching the WebPage since it was last successfully fetched. * @param value the value to set.
*/
- public boolean isRetriesSinceFetchDirty(Integer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isRetriesSinceFetchDirty(java.lang.Integer value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'modifiedTime' field.
- */
- public Long getModifiedTime() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. */
+ public java.lang.Long getModifiedTime() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'modifiedTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. * @param value the value to set.
*/
- public void setModifiedTime(Long value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setModifiedTime(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'modifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was modified by the WebPage author, if this is not available we default to the server for this information. This is important to understand the changing nature of the WebPage. * @param value the value to set.
*/
- public boolean isModifiedTimeDirty(Long value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isModifiedTimeDirty(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'prevModifiedTime' field.
- */
- public Long getPrevModifiedTime() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. */
+ public java.lang.Long getPrevModifiedTime() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'prevModifiedTime' field.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. * @param value the value to set.
*/
- public void setPrevModifiedTime(Long value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setPrevModifiedTime(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'prevModifiedTime' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The system time in milliseconds for when this WebPage was previously modified by the author, if this is not available then we default to the server for this information. This is important to understand the changing nature of a WebPage. * @param value the value to set.
*/
- public boolean isPrevModifiedTimeDirty(Long value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isPrevModifiedTimeDirty(java.lang.Long value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'protocolStatus' field.
*/
- public ProtocolStatus getProtocolStatus() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ public org.apache.nutch.storage.ProtocolStatus getProtocolStatus() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
@@ -1815,8 +1825,8 @@
* Sets the value of the 'protocolStatus' field.
* @param value the value to set.
*/
- public void setProtocolStatus(ProtocolStatus value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setProtocolStatus(org.apache.nutch.storage.ProtocolStatus value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
@@ -1823,153 +1833,153 @@
* Checks the dirty status of the 'protocolStatus' field. A field is dirty if it represents a change that has not yet been written to the database.
* @param value the value to set.
*/
- public boolean isProtocolStatusDirty(ProtocolStatus value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isProtocolStatusDirty(org.apache.nutch.storage.ProtocolStatus value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'content' field.
- */
+ * The entire raw document content e.g. raw XHTML */
public java.nio.ByteBuffer getContent() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'content' field.
- * @param value the value to set.
+ * The entire raw document content e.g. raw XHTML * @param value the value to set.
*/
public void setContent(java.nio.ByteBuffer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'content' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The entire raw document content e.g. raw XHTML * @param value the value to set.
*/
public boolean isContentDirty(java.nio.ByteBuffer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'contentType' field.
- */
- public CharSequence getContentType() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. */
+ public java.lang.CharSequence getContentType() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'contentType' field.
- * @param value the value to set.
+ * The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. * @param value the value to set.
*/
- public void setContentType(CharSequence value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setContentType(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'contentType' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The type of the content contained within the document itself. ContentType is an alias for MimeType. Historically, this parameter was only called MimeType, but since this is actually the value included in the HTTP Content-Type header, it can also include the character set encoding, which makes it more than just a MimeType specification. If MimeType is specified e.g. not None, that value is used. Otherwise, ContentType is used. If neither is given, the DEFAULT_CONTENT_TYPE setting is used. * @param value the value to set.
*/
- public boolean isContentTypeDirty(CharSequence value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isContentTypeDirty(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'prevSignature' field.
- */
+ * An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. */
public java.nio.ByteBuffer getPrevSignature() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'prevSignature' field.
- * @param value the value to set.
+ * An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. * @param value the value to set.
*/
public void setPrevSignature(java.nio.ByteBuffer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'prevSignature' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * An implementation of a WebPage's previous signature from which it can be identified and referenced at any point in time. This can be used to uniquely identify WebPage deltas based on page fingerprints. * @param value the value to set.
*/
public boolean isPrevSignatureDirty(java.nio.ByteBuffer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'signature' field.
- */
+ * An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. */
public java.nio.ByteBuffer getSignature() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'signature' field.
- * @param value the value to set.
+ * An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. * @param value the value to set.
*/
public void setSignature(java.nio.ByteBuffer value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'signature' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * An implementation of a WebPage's signature from which it can be identified and referenced at any point in time. This is essentially the WebPage's fingerprint represnting its state for any point in time. * @param value the value to set.
*/
public boolean isSignatureDirty(java.nio.ByteBuffer value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'title' field.
- */
- public CharSequence getTitle() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The title of the WebPage. */
+ public java.lang.CharSequence getTitle() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'title' field.
- * @param value the value to set.
+ * The title of the WebPage. * @param value the value to set.
*/
- public void setTitle(CharSequence value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setTitle(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'title' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The title of the WebPage. * @param value the value to set.
*/
- public boolean isTitleDirty(CharSequence value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isTitleDirty(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'text' field.
- */
- public CharSequence getText() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * The textual content of the WebPage devoid from native markup. */
+ public java.lang.CharSequence getText() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'text' field.
- * @param value the value to set.
+ * The textual content of the WebPage devoid from native markup. * @param value the value to set.
*/
- public void setText(CharSequence value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setText(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'text' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * The textual content of the WebPage devoid from native markup. * @param value the value to set.
*/
- public boolean isTextDirty(CharSequence value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isTextDirty(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'parseStatus' field.
*/
- public ParseStatus getParseStatus() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ public org.apache.nutch.storage.ParseStatus getParseStatus() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
@@ -1976,8 +1986,8 @@
* Sets the value of the 'parseStatus' field.
* @param value the value to set.
*/
- public void setParseStatus(ParseStatus value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setParseStatus(org.apache.nutch.storage.ParseStatus value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
@@ -1984,193 +1994,196 @@
* Checks the dirty status of the 'parseStatus' field. A field is dirty if it represents a change that has not yet been written to the database.
* @param value the value to set.
*/
- public boolean isParseStatusDirty(ParseStatus value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isParseStatusDirty(org.apache.nutch.storage.ParseStatus value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'score' field.
- */
- public Float getScore() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. */
+ public java.lang.Float getScore() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'score' field.
- * @param value the value to set.
+ * A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. * @param value the value to set.
*/
- public void setScore(Float value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setScore(java.lang.Float value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'score' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * A score used to determine a WebPage's relevance within the web graph it is part of. This score may change over time based on graph characteristics. * @param value the value to set.
*/
- public boolean isScoreDirty(Float value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isScoreDirty(java.lang.Float value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'reprUrl' field.
- */
- public CharSequence getReprUrl() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler */
+ public java.lang.CharSequence getReprUrl() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'reprUrl' field.
- * @param value the value to set.
+ * In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler * @param value the value to set.
*/
- public void setReprUrl(CharSequence value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setReprUrl(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'reprUrl' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * In the case where we are given two urls, a source and a destination of a redirect, we should determine and persist the representative url. The logic used to determine this is based largely on Yahoo!'s Slurp Crawler * @param value the value to set.
*/
- public boolean isReprUrlDirty(CharSequence value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isReprUrlDirty(java.lang.CharSequence value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'headers' field.
- */
- public java.util.Map getHeaders() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. */
+ public java.util.Map getHeaders() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'headers' field.
- * @param value the value to set.
+ * Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. * @param value the value to set.
*/
- public void setHeaders(java.util.Map value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setHeaders(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'headers' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Header information returned from the web server used to server the content which is subsequently fetched from. This includes keys such as TRANSFER_ENCODING, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_LOCATION, CONTENT_DISPOSITION, CONTENT_MD5, CONTENT_TYPE, LAST_MODIFIED and LOCATION. * @param value the value to set.
*/
- public boolean isHeadersDirty(java.util.Map value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isHeadersDirty(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'outlinks' field.
- */
- public java.util.Map getOutlinks() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Embedded hyperlinks which direct outside of the current domain. */
+ public java.util.Map getOutlinks() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'outlinks' field.
- * @param value the value to set.
+ * Embedded hyperlinks which direct outside of the current domain. * @param value the value to set.
*/
- public void setOutlinks(java.util.Map value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setOutlinks(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones");
}
/**
* Checks the dirty status of the 'outlinks' field. A field is dirty if it represents a change that has not yet been written to the database.
- * @param value the value to set.
+ * Embedded hyperlinks which direct outside of the current domain. * @param value the value to set.
*/
- public boolean isOutlinksDirty(java.util.Map value) {
- throw new UnsupportedOperationException("IsDirty is not supported on tombstones");
+ public boolean isOutlinksDirty(java.util.Map value) {
+ throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones");
}
/**
* Gets the value of the 'inlinks' field.
- */
- public java.util.Map getInlinks() {
- throw new UnsupportedOperationException("Get is not supported on tombstones");
+ * Embedded hyperlinks which link to pages within the current domain. */
+ public java.util.Map getInlinks() {
+ throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones");
}
/**
* Sets the value of the 'inlinks' field.
- * @param value the value to set.
+ * Embedded hyperlinks which link to pages within the current domain. * @param value the value to set.
*/
- public void setInlinks(java.util.Map value) {
- throw new UnsupportedOperationException("Set is not supported on tombstones");
+ public void setInlinks(java.util.Map