Constant Field Values

Contents

• crawlercommons.domains.*
• crawlercommons.fetcher.*
• crawlercommons.robots.*
• crawlercommons.sitemaps.*

crawlercommons.domains.*

crawlercommons.domains.EffectiveTldFinder

Modifier and Type	Constant Field	Value
public static final String	COMMENT	"//"
public static final char	DOT	46
public static final String	DOT_REGEX	"\\."
public static final String	ETLD_DATA	"/effective_tld_names.dat"
public static final String	EXCEPTION	"!"
public static final String	WILD_CARD	"*."

crawlercommons.fetcher.*

crawlercommons.fetcher.BaseFetcher

Modifier and Type	Constant Field	Value
public static final int	DEFAULT_MAX_CONTENT_SIZE	65536

crawlercommons.fetcher.http.BaseHttpFetcher

Modifier and Type	Constant Field	Value
public static final String	DEFAULT_ACCEPT_LANGUAGE	"en-us,en-gb,en;q=0.7,*;q=0.3"
public static final int	DEFAULT_MAX_CONNECTIONS_PER_HOST	2
public static final int	DEFAULT_MAX_REDIRECTS	20
public static final int	DEFAULT_MIN_RESPONSE_RATE	-2147483648
public static final int	NO_MIN_RESPONSE_RATE	-2147483648
public static final int	NO_REDIRECTS	0

crawlercommons.fetcher.http.UserAgent

Modifier and Type	Constant Field	Value
public static final String	DEFAULT_BROWSER_VERSION	"Mozilla/5.0"

crawlercommons.robots.*

crawlercommons.robots.BaseRobotRules

Modifier and Type	Constant Field	Value
public static final long	UNSET_CRAWL_DELAY	-9223372036854775808L

crawlercommons.sitemaps.*

crawlercommons.sitemaps.SiteMapParser

Modifier and Type	Constant Field	Value
public static final int	MAX_BYTES_ALLOWED	10485760

crawlercommons.sitemaps.SiteMapURL

Modifier and Type	Constant Field	Value
public static final double	DEFAULT_PRIORITY	0.5