Constant Field Values

Contents

• com.digitalpebble.*

com.digitalpebble.*

com.digitalpebble.stormcrawler.Constants

Modifier and Type	Constant Field	Value
public static final String	AllowRedirParamName	"redirections.allowed"
public static final String	defaultFetchIntervalParamName	"fetchInterval.default"
public static final String	DELETION_STREAM_NAME	"deletion"
public static final String	errorFetchIntervalParamName	"fetchInterval.error"
public static final String	fetchErrorCountParamName	"fetch.error.count"
public static final String	fetchErrorFetchIntervalParamName	"fetchInterval.fetch.error"
public static final int	MAX_ARRAY_SIZE	2147483639
public static final String	PARTITION_MODE_DOMAIN	"byDomain"
public static final String	PARTITION_MODE_HOST	"byHost"
public static final String	PARTITION_MODE_IP	"byIP"
public static final String	PARTITION_MODEParamName	"partition.url.mode"
public static final String	STATUS_ERROR_CAUSE	"error.cause"
public static final String	STATUS_ERROR_MESSAGE	"error.message"
public static final String	STATUS_ERROR_SOURCE	"error.source"
public static final String	StatusStreamName	"status"

com.digitalpebble.stormcrawler.bolt.FeedParserBolt

Modifier and Type	Constant Field	Value
public static final String	isFeedKey	"isFeed"

com.digitalpebble.stormcrawler.bolt.FetcherBolt

Modifier and Type	Constant Field	Value
public static final String	QUEUED_TIMEOUT_PARAM_KEY	"fetcher.timeout.queue"

com.digitalpebble.stormcrawler.bolt.JSoupParserBolt

Modifier and Type	Constant Field	Value
public static final String	ANCHORS_KEY_NAME	"anchors"

com.digitalpebble.stormcrawler.bolt.SimpleFetcherBolt

Modifier and Type	Constant Field	Value
public static final String	QUEUE_MODE_DOMAIN	"byDomain"
public static final String	QUEUE_MODE_HOST	"byHost"
public static final String	QUEUE_MODE_IP	"byIP"
public static final String	THROTTLE_STREAM	"throttle"

com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt

Modifier and Type	Constant Field	Value
public static final String	foundSitemapKey	"foundSitemap"
public static final String	isSitemapKey	"isSitemap"

com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt

Modifier and Type	Constant Field	Value
public static final String	canonicalMetadataParamName	"indexer.canonical.name"
public static final String	ignoreEmptyFieldValueParamName	"indexer.ignore.empty.fields"
public static final String	metadata2fieldParamName	"indexer.md.mapping"
public static final String	metadataFilterParamName	"indexer.md.filter"
public static final String	textFieldParamName	"indexer.text.fieldname"
public static final String	textLengthParamName	"indexer.text.maxlength"
public static final String	urlFieldParamName	"indexer.url.fieldname"

com.digitalpebble.stormcrawler.parse.TextExtractor

Modifier and Type	Constant Field	Value
public static final String	EXCLUDE_PARAM_NAME	"textextractor.exclude.tags"
public static final String	INCLUDE_PARAM_NAME	"textextractor.include.pattern"
public static final String	NO_TEXT_PARAM_NAME	"textextractor.no.text"
public static final String	TEXT_MAX_TEXT_PARAM_NAME	"textextractor.skip.after"

com.digitalpebble.stormcrawler.persistence.AbstractQueryingSpout

Modifier and Type	Constant Field	Value
protected static final String	resetFetchDateParamName	"spout.reset.fetchdate.after"
protected static final String	StatusMaxDelayParamName	"spout.max.delay.queries"
protected static final String	StatusMinDelayParamName	"spout.min.delay.queries"
protected static final String	StatusTTLPurgatory	"spout.ttl.purgatory"

com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt

Modifier and Type	Constant Field	Value
public static final String	AS_IS_NEXTFETCHDATE_METADATA	"status.store.as.is.with.nextfetchdate"

com.digitalpebble.stormcrawler.persistence.AdaptiveScheduler

Modifier and Type	Constant Field	Value
public static final String	FETCH_INTERVAL_KEY	"fetchInterval"
public static final String	INTERVAL_DEC_RATE	"scheduler.adaptive.fetchInterval.rate.decr"
public static final String	INTERVAL_INC_RATE	"scheduler.adaptive.fetchInterval.rate.incr"
public static final String	INTERVAL_MAX	"scheduler.adaptive.fetchInterval.max"
public static final String	INTERVAL_MIN	"scheduler.adaptive.fetchInterval.min"
public static final String	SET_LAST_MODIFIED	"scheduler.adaptive.setLastModified"
public static final String	SIGNATURE_KEY	"signature"
public static final String	SIGNATURE_MODIFIED_KEY	"signatureChangeDate"
public static final String	SIGNATURE_OLD_KEY	"signatureOld"

com.digitalpebble.stormcrawler.persistence.DefaultScheduler

Modifier and Type	Constant Field	Value
public static final String	DELAY_METADATA	"scheduler.delay.mins"

com.digitalpebble.stormcrawler.persistence.Scheduler

Modifier and Type	Constant Field	Value
public static final String	schedulerClassParamName	"scheduler.class"

com.digitalpebble.stormcrawler.persistence.urlbuffer.SchedulingURLBuffer

Modifier and Type	Constant Field	Value
public static final String	MAXTIMEPARAM	"priority.buffer.max.time.msec"

com.digitalpebble.stormcrawler.persistence.urlbuffer.URLBuffer

Modifier and Type	Constant Field	Value
public static final String	bufferClassParamName	"urlbuffer.class"

com.digitalpebble.stormcrawler.protocol.AbstractHttpProtocol

Modifier and Type	Constant Field	Value
protected static final String	RESPONSE_COOKIES_HEADER	"set-cookie"
protected static final String	SET_HEADER_BY_REQUEST	"set-header"

com.digitalpebble.stormcrawler.protocol.HttpHeaders

Modifier and Type	Constant Field	Value
public static final String	CONTENT_DISPOSITION	"content-disposition"
public static final String	CONTENT_ENCODING	"content-encoding"
public static final String	CONTENT_LANGUAGE	"content-language"
public static final String	CONTENT_LENGTH	"content-length"
public static final String	CONTENT_LOCATION	"content-location"
public static final String	CONTENT_MD5	"content-md5"
public static final String	CONTENT_TYPE	"content-type"
public static final String	LAST_MODIFIED	"last-modified"
public static final String	LOCATION	"location"
public static final String	TRANSFER_ENCODING	"transfer-encoding"

com.digitalpebble.stormcrawler.protocol.ProtocolResponse

Modifier and Type	Constant Field	Value
public static final String	PROTOCOL_MD_PREFIX_PARAM	"protocol.md.prefix"
public static final String	PROTOCOL_VERSIONS_KEY	"_protocol_versions_"
public static final String	REQUEST_HEADERS_KEY	"_request.headers_"
public static final String	REQUEST_TIME_KEY	"_request.time_"
public static final String	RESPONSE_HEADERS_KEY	"_response.headers_"
public static final String	RESPONSE_IP_KEY	"_response.ip_"
public static final String	TRIMMED_RESPONSE_KEY	"http.trimmed"
public static final String	TRIMMED_RESPONSE_REASON_KEY	"http.trimmed.reason"

com.digitalpebble.stormcrawler.protocol.RobotRulesParser

Modifier and Type	Constant Field	Value
public static final String	cacheConfigParamName	"robots.cache.spec"
public static final String	errorcacheConfigParamName	"robots.error.cache.spec"

com.digitalpebble.stormcrawler.protocol.selenium.DelegatorRemoteDriverProtocol

Modifier and Type	Constant Field	Value
public static final String	PROTOCOL_IMPL_CONFIG	"selenium.delegated.protocol"
public static final String	USE_SELENIUM_KEY	"protocol.use.selenium"

com.digitalpebble.stormcrawler.spout.FileSpout

Modifier and Type	Constant Field	Value
public static final int	BATCH_SIZE	10000

com.digitalpebble.stormcrawler.util.MetadataTransfer

Modifier and Type	Constant Field	Value
public static final String	depthKeyName	"depth"
public static final String	maxDepthKeyName	"max.depth"
public static final String	metadataPersistParamName	"metadata.persist"
public static final String	metadataTransferClassParamName	"metadata.transfer.class"
public static final String	metadataTransferParamName	"metadata.transfer"
public static final String	trackDepthParamName	"metadata.track.depth"
public static final String	trackPathParamName	"metadata.track.path"
public static final String	urlPathKeyName	"url.path"

com.digitalpebble.stormcrawler.util.RobotsTags

Modifier and Type	Constant Field	Value
public static final String	ROBOTS_NO_CACHE	"robots.noCache"
public static final String	ROBOTS_NO_FOLLOW	"robots.noFollow"
public static final String	ROBOTS_NO_FOLLOW_STRICT	"robots.noFollow.strict"
public static final String	ROBOTS_NO_INDEX	"robots.noIndex"