python-scrapy-doc 0.14.4-1 / usr / share / doc / python-scrapy-doc / html / searchindex.js

Search.setIndex({objects:{"":{CLOSESPIDER_ITEMCOUNT:[17,1,1,"std:setting-CLOSESPIDER_ITEMCOUNT"],update_telnet_vars:[27,2,1,"std:signal-update_telnet_vars"],DEFAULT_RESPONSE_ENCODING:[5,1,1,"std:setting-DEFAULT_RESPONSE_ENCODING"],dont_retry:[19,3,1,"std:reqmeta-dont_retry"],RETRY_HTTP_CODES:[19,1,1,"std:setting-RETRY_HTTP_CODES"],HTTPCACHE_DIR:[19,1,1,"std:setting-HTTPCACHE_DIR"],MEMUSAGE_REPORT:[5,1,1,"std:setting-MEMUSAGE_REPORT"],DOWNLOAD_HANDLERS_BASE:[5,1,1,"std:setting-DOWNLOAD_HANDLERS_BASE"],MAIL_FROM:[31,1,1,"std:setting-MAIL_FROM"],HTTPCACHE_EXPIRATION_SECS:[19,1,1,"std:setting-HTTPCACHE_EXPIRATION_SECS"],stats_spider_closed:[38,2,1,"std:signal-stats_spider_closed"],DOWNLOAD_TIMEOUT:[5,1,1,"std:setting-DOWNLOAD_TIMEOUT"],genspider:[34,0,1,"std:command-genspider"],MAIL_PASS:[31,1,1,"std:setting-MAIL_PASS"],MEMUSAGE_LIMIT_MB:[5,1,1,"std:setting-MEMUSAGE_LIMIT_MB"],EXTENSIONS:[5,1,1,"std:setting-EXTENSIONS"],DEPTH_PRIORITY:[5,1,1,"std:setting-DEPTH_PRIORITY"],dont_redirect:[19,3,1,"std:reqmeta-dont_redirect"],ENCODING_ALIASES_BASE:[5,1,1,"std:setting-ENCODING_ALIASES_BASE"],TELNETCONSOLE_HOST:[27,1,1,"std:setting-TELNETCONSOLE_HOST"],WEBSERVICE_HOST:[18,1,1,"std:setting-WEBSERVICE_HOST"],mail_sent:[31,2,1,"std:signal-mail_sent"],spider_error:[30,2,1,"std:signal-spider_error"],IMAGES_MIN_WIDTH:[13,1,1,"std:setting-IMAGES_MIN_WIDTH"],SPIDER_MODULES:[5,1,1,"std:setting-SPIDER_MODULES"],RETRY_TIMES:[19,1,1,"std:setting-RETRY_TIMES"],close_spider:[2,6,1,""],TELNETCONSOLE_PORT:[27,1,1,"std:setting-TELNETCONSOLE_PORT"],TELNETCONSOLE_ENABLED:[5,1,1,"std:setting-TELNETCONSOLE_ENABLED"],DOWNLOADER_MIDDLEWARES:[5,1,1,"std:setting-DOWNLOADER_MIDDLEWARES"],item_dropped:[30,2,1,"std:signal-item_dropped"],HTTPCACHE_DBM_MODULE:[19,1,1,"std:setting-HTTPCACHE_DBM_MODULE"],ROBOTSTXT_OBEY:[5,1,1,"std:setting-ROBOTSTXT_OBEY"],DEPTH_LIMIT:[5,1,1,"std:setting-DEPTH_LIMIT"],settings:[34,0,1,"std:command-settings"],edit:[34,0,1,"std:command-edit"],request_received:[30,2,1,"std:signal-request_received"],list:[34,0,1,"std:command-list"],server:[34,0,1,"std:command-server"],CLOSESPIDER_PAGECOUNT:[17,1,1,"std:setting-CLOSESPIDER_PAGECOUNT"],stats_spider_closing:[38,2,1,"std:signal-stats_spider_closing"],view:[34,0,1,"std:command-view"],URLLENGTH_LIMIT:[5,1,1,"std:setting-URLLENGTH_LIMIT"],FEED_EXPORTERS:[7,1,1,"std:setting-FEED_EXPORTERS"],LOG_ENCODING:[5,1,1,"std:setting-LOG_ENCODING"],FEED_EXPORTERS_BASE:[7,1,1,"std:setting-FEED_EXPORTERS_BASE"],DOWNLOADER_DEBUG:[5,1,1,"std:setting-DOWNLOADER_DEBUG"],FEED_FORMAT:[7,1,1,"std:setting-FEED_FORMAT"],HTTPCACHE_IGNORE_SCHEMES:[19,1,1,"std:setting-HTTPCACHE_IGNORE_SCHEMES"],spider_idle:[30,2,1,"std:signal-spider_idle"],MEMDEBUG_ENABLED:[5,1,1,"std:setting-MEMDEBUG_ENABLED"],DNSCACHE_ENABLED:[5,1,1,"std:setting-DNSCACHE_ENABLED"],response_downloaded:[30,2,1,"std:signal-response_downloaded"],CONCURRENT_REQUESTS_PER_IP:[5,1,1,"std:setting-CONCURRENT_REQUESTS_PER_IP"],version:[34,0,1,"std:command-version"],EDITOR:[5,1,1,"std:setting-EDITOR"],MAIL_HOST:[31,1,1,"std:setting-MAIL_HOST"],CONCURRENT_REQUESTS:[5,1,1,"std:setting-CONCURRENT_REQUESTS"],STATS_ENABLED:[5,1,1,"std:setting-STATS_ENABLED"],CLOSESPIDER_ERRORCOUNT:[17,1,1,"std:setting-CLOSESPIDER_ERRORCOUNT"],COMMANDS_MODULE:[34,1,1,"std:setting-COMMANDS_MODULE"],STATS_CLASS:[5,1,1,"std:setting-STATS_CLASS"],item_scraped:[30,2,1,"std:signal-item_scraped"],open_spider:[2,6,1,""],response_received:[30,2,1,"std:signal-response_received"],FEED_STORAGES_BASE:[7,1,1,"std:setting-FEED_STORAGES_BASE"],REDIRECT_ENABLED:[19,1,1,"std:setting-REDIRECT_ENABLED"],SPIDER_MIDDLEWARES:[5,1,1,"std:setting-SPIDER_MIDDLEWARES"],AWS_ACCESS_KEY_ID:[5,1,1,"std:setting-AWS_ACCESS_KEY_ID"],NEWSPIDER_MODULE:[5,1,1,"std:setting-NEWSPIDER_MODULE"],DEPTH_STATS_VERBOSE:[5,1,1,"std:setting-DEPTH_STATS_VERBOSE"],CONCURRENT_ITEMS:[5,1,1,"std:setting-CONCURRENT_ITEMS"],WEBSERVICE_ENABLED:[18,1,1,"std:setting-WEBSERVICE_ENABLED"],WEBSERVICE_PORT:[18,1,1,"std:setting-WEBSERVICE_PORT"],ENCODING_ALIASES:[5,1,1,"std:setting-ENCODING_ALIASES"],MAIL_PORT:[31,1,1,"std:setting-MAIL_PORT"],STATS_DUMP:[5,1,1,"std:setting-STATS_DUMP"],MEMUSAGE_NOTIFY_MAIL:[5,1,1,"std:setting-MEMUSAGE_NOTIFY_MAIL"],DOWNLOAD_HANDLERS:[5,1,1,"std:setting-DOWNLOAD_HANDLERS"],IMAGES_MIN_HEIGHT:[13,1,1,"std:setting-IMAGES_MIN_HEIGHT"],redirect_urls:[19,3,1,"std:reqmeta-redirect_urls"],LOG_LEVEL:[5,1,1,"std:setting-LOG_LEVEL"],spider_closed:[30,2,1,"std:signal-spider_closed"],handle_httpstatus_list:[29,3,1,"std:reqmeta-handle_httpstatus_list"],REDIRECT_MAX_TIMES:[5,1,1,"std:setting-REDIRECT_MAX_TIMES"],REDIRECT_PRIORITY_ADJUST:[5,1,1,"std:setting-REDIRECT_PRIORITY_ADJUST"],DUPEFILTER_CLASS:[5,1,1,"std:setting-DUPEFILTER_CLASS"],RETRY_ENABLED:[19,1,1,"std:setting-RETRY_ENABLED"],HTTPCACHE_ENABLED:[19,1,1,"std:setting-HTTPCACHE_ENABLED"],LOG_ENABLED:[5,1,1,"std:setting-LOG_ENABLED"],MAIL_USER:[31,1,1,"std:setting-MAIL_USER"],DEFAULT_ITEM_CLASS:[5,1,1,"std:setting-DEFAULT_ITEM_CLASS"],DEPTH_STATS:[5,1,1,"std:setting-DEPTH_STATS"],shell:[34,0,1,"std:command-shell"],BOT_VERSION:[5,1,1,"std:setting-BOT_VERSION"],runspider:[34,0,1,"std:command-runspider"],EXTENSIONS_BASE:[5,1,1,"std:setting-EXTENSIONS_BASE"],FEED_STORAGES:[7,1,1,"std:setting-FEED_STORAGES"],BOT_NAME:[5,1,1,"std:setting-BOT_NAME"],startproject:[34,0,1,"std:command-startproject"],crawl:[34,0,1,"std:command-crawl"],CONCURRENT_REQUESTS_PER_DOMAIN:[5,1,1,"std:setting-CONCURRENT_REQUESTS_PER_DOMAIN"],HTTPCACHE_IGNORE_HTTP_CODES:[19,1,1,"std:setting-HTTPCACHE_IGNORE_HTTP_CODES"],WEBSERVICE_LOGFILE:[18,1,1,"std:setting-WEBSERVICE_LOGFILE"],engine_stopped:[30,2,1,"std:signal-engine_stopped"],MEMUSAGE_WARNING_MB:[5,1,1,"std:setting-MEMUSAGE_WARNING_MB"],MEMDEBUG_NOTIFY:[5,1,1,"std:setting-MEMDEBUG_NOTIFY"],FEED_STORE_EMPTY:[7,1,1,"std:setting-FEED_STORE_EMPTY"],fetch:[34,0,1,"std:command-fetch"],COOKIES_DEBUG:[19,1,1,"std:setting-COOKIES_DEBUG"],FEED_URI:[7,1,1,"std:setting-FEED_URI"],USER_AGENT:[5,1,1,"std:setting-USER_AGENT"],parse:[34,0,1,"std:command-parse"],COOKIES_ENABLED:[19,1,1,"std:setting-COOKIES_ENABLED"],process_item:[2,6,1,""],ITEM_PIPELINES:[5,1,1,"std:setting-ITEM_PIPELINES"],SPIDER_MIDDLEWARES_BASE:[5,1,1,"std:setting-SPIDER_MIDDLEWARES_BASE"],HTTPCACHE_IGNORE_MISSING:[19,1,1,"std:setting-HTTPCACHE_IGNORE_MISSING"],IMAGES_EXPIRES:[13,1,1,"std:setting-IMAGES_EXPIRES"],IMAGES_THUMBS:[13,1,1,"std:setting-IMAGES_THUMBS"],AWS_SECRET_ACCESS_KEY:[5,1,1,"std:setting-AWS_SECRET_ACCESS_KEY"],deploy:[34,0,1,"std:command-deploy"],RANDOMIZE_DOWNLOAD_DELAY:[5,1,1,"std:setting-RANDOMIZE_DOWNLOAD_DELAY"],TEMPLATES_DIR:[5,1,1,"std:setting-TEMPLATES_DIR"],LOG_STDOUT:[5,1,1,"std:setting-LOG_STDOUT"],DOWNLOADER_STATS:[5,1,1,"std:setting-DOWNLOADER_STATS"],LOG_FILE:[5,1,1,"std:setting-LOG_FILE"],HTTPCACHE_STORAGE:[19,1,1,"std:setting-HTTPCACHE_STORAGE"],DOWNLOADER_MIDDLEWARES_BASE:[5,1,1,"std:setting-DOWNLOADER_MIDDLEWARES_BASE"],REDIRECT_MAX_METAREFRESH_DELAY:[5,1,1,"std:setting-REDIRECT_MAX_METAREFRESH_DELAY"],engine_started:[30,2,1,"std:signal-engine_started"],stats_spider_opened:[38,2,1,"std:signal-stats_spider_opened"],spider_opened:[30,2,1,"std:signal-spider_opened"],DEFAULT_REQUEST_HEADERS:[5,1,1,"std:setting-DEFAULT_REQUEST_HEADERS"],CLOSESPIDER_TIMEOUT:[17,1,1,"std:setting-CLOSESPIDER_TIMEOUT"],IMAGES_STORE:[13,1,1,"std:setting-IMAGES_STORE"],MEMUSAGE_ENABLED:[5,1,1,"std:setting-MEMUSAGE_ENABLED"],SCHEDULER:[5,1,1,"std:setting-SCHEDULER"],STATSMAILER_RCPTS:[5,1,1,"std:setting-STATSMAILER_RCPTS"],DOWNLOAD_DELAY:[5,1,1,"std:setting-DOWNLOAD_DELAY"]},"scrapy.contrib.spidermiddleware.referer":{RefererMiddleware:[29,5,1,""]},"scrapy.contrib.spidermiddleware.SpiderMiddleware":{process_spider_input:[29,6,1,""],process_spider_output:[29,6,1,""],process_spider_exception:[29,6,1,""]},"scrapy.contrib.webservice.enginestatus.scrapy.webservice.JsonRpcResource":{get_target:[18,6,1,""]},"scrapy.contrib.downloadermiddleware.robotstxt":{RobotsTxtMiddleware:[19,5,1,""]},"scrapy.conf":{Settings:[5,5,1,""]},"scrapy.statscol.MemoryStatsCollector":{spider_stats:[38,9,1,""]},"scrapy.conf.Settings":{getint:[5,6,1,""],getfloat:[5,6,1,""],getbool:[5,6,1,""],getlist:[5,6,1,""],get:[5,6,1,""]},"scrapy.selector.XPathSelectorList":{re:[24,6,1,""],extract:[24,6,1,""],select:[24,6,1,""],extract_unquoted:[24,6,1,""]},"scrapy.http.Response":{status:[26,9,1,""],body:[26,9,1,""],url:[26,9,1,""],request:[26,9,1,""],replace:[26,6,1,""],headers:[26,9,1,""],meta:[26,9,1,""],flags:[26,9,1,""],copy:[26,6,1,""]},"scrapy.contrib.loader.XPathItemLoader":{add_xpath:[0,6,1,""],default_selector_class:[0,9,1,""],selector:[0,9,1,""],get_xpath:[0,6,1,""],replace_xpath:[0,6,1,""]},"scrapy.contrib.spiders.CrawlSpider":{rules:[20,9,1,""]},"scrapy.http.Request":{body:[26,9,1,""],url:[26,9,1,""],replace:[26,6,1,""],headers:[26,9,1,""],meta:[26,9,1,""],copy:[26,6,1,""],method:[26,9,1,""]},"scrapy.contrib.spidermiddleware.urllength":{UrlLengthMiddleware:[29,5,1,""]},"scrapy.contrib.downloadermiddleware.httpcompression":{HttpCompressionMiddleware:[19,5,1,""]},"scrapy.contrib.spidermiddleware":{offsite:[29,4,1,""],depth:[29,4,1,""],SpiderMiddleware:[29,5,1,""],referer:[29,4,1,""],urllength:[29,4,1,""],httperror:[29,4,1,""]},"scrapy.statscol.StatsCollector":{get_stats:[38,6,1,""],min_value:[38,6,1,""],max_value:[38,6,1,""],iter_spider_stats:[38,6,1,""],get_value:[38,6,1,""],inc_value:[38,6,1,""],close_spider:[38,6,1,""],engine_stopped:[38,6,1,""],open_spider:[38,6,1,""],clear_stats:[38,6,1,""],set_stats:[38,6,1,""],set_value:[38,6,1,""]},"scrapy.item":{Field:[8,5,1,""],Item:[8,5,1,""]},"scrapy.contrib.memdebug.scrapy.contrib.memdebug":{MemoryDebugger:[17,5,1,""]},"scrapy.contrib.loader.ItemLoader":{context:[0,9,1,""],get_value:[0,6,1,""],add_value:[0,6,1,""],get_output_value:[0,6,1,""],get_output_processor:[0,6,1,""],default_input_processor:[0,9,1,""],get_input_processor:[0,6,1,""],get_collected_values:[0,6,1,""],replace_value:[0,6,1,""],item:[0,9,1,""],default_item_class:[0,9,1,""],default_output_processor:[0,9,1,""],load_item:[0,6,1,""]},"scrapy.contrib.linkextractors.sgml":{SgmlLinkExtractor:[25,5,1,""],BaseSgmlLinkExtractor:[25,5,1,""]},"scrapy.contrib.spidermiddleware.offsite":{OffsiteMiddleware:[29,5,1,""]},"scrapy.contrib.webservice.enginestatus.scrapy.webservice":{JsonResource:[18,5,1,""],JsonRpcResource:[18,5,1,""]},"scrapy.contrib.memusage.scrapy.contrib.memusage":{MemoryUsage:[17,5,1,""]},"scrapy.contrib.downloadermiddleware.useragent":{UserAgentMiddleware:[19,5,1,""]},"scrapy.extension.ExtensionManager":{load:[17,6,1,""],reload:[17,6,1,""],enabled:[17,9,1,""],disabled:[17,9,1,""],loaded:[17,9,1,""]},"scrapy.statscol":{MemoryStatsCollector:[38,5,1,""],StatsCollector:[38,5,1,""],DummyStatsCollector:[38,5,1,""]},"scrapy.contrib.downloadermiddleware.downloadtimeout":{DownloadTimeoutMiddleware:[19,5,1,""]},"scrapy.contrib.spiders":{XMLFeedSpider:[20,5,1,""],CrawlSpider:[20,5,1,""],CSVFeedSpider:[20,5,1,""],Rule:[20,5,1,""],SitemapSpider:[20,5,1,""]},"scrapy.log":{INFO:[37,10,1,""],WARNING:[37,10,1,""],started:[37,9,1,""],start:[37,7,1,""],CRITICAL:[37,10,1,""],ERROR:[37,10,1,""],DEBUG:[37,10,1,""],msg:[37,7,1,""]},"scrapy.contrib.downloadermiddleware.defaultheaders":{DefaultHeadersMiddleware:[19,5,1,""]},"scrapy.contrib.webservice.enginestatus.scrapy.webservice.JsonResource":{ws_name:[18,9,1,""]},"scrapy.contrib.spiders.SitemapSpider":{sitemap_urls:[20,9,1,""],sitemap_rules:[20,9,1,""],sitemap_follow:[20,9,1,""]},scrapy:{http:[26,4,1,""],webservice:[17,4,1,""],extension:[17,4,1,""],statscol:[38,4,1,""],spider:[20,4,1,""],telnet:[27,4,1,""],selector:[24,4,1,""],signals:[30,4,1,""],item:[8,4,1,""],conf:[5,4,1,""],exceptions:[22,4,1,""],mail:[31,4,1,""],log:[37,4,1,""]},"scrapy.contrib.downloadermiddleware.retry":{RetryMiddleware:[19,5,1,""]},"scrapy.contrib.pipeline.images.ImagesPipeline":{get_media_requests:[13,6,1,""],item_completed:[13,6,1,""]},"scrapy.selector":{XPathSelectorList:[24,5,1,""],HtmlXPathSelector:[24,5,1,""],XmlXPathSelector:[24,5,1,""],XPathSelector:[24,5,1,""]},"scrapy.telnet.scrapy.telnet":{TelnetConsole:[17,5,1,""]},"scrapy.http.TextResponse":{body_as_unicode:[26,6,1,""],encoding:[26,9,1,""]},"scrapy.contrib.linkextractors":{sgml:[25,4,1,""]},"scrapy.utils":{trackref:[28,4,1,""]},"scrapy.contrib.spidermiddleware.depth":{DepthMiddleware:[29,5,1,""]},"scrapy.contrib.debug.scrapy.contrib.debug":{Debugger:[17,5,1,""],StackTraceDump:[17,5,1,""]},"scrapy.telnet":{update_telnet_vars:[27,7,1,""]},"scrapy.contrib.downloadermiddleware.redirect":{RedirectMiddleware:[19,5,1,""]},"scrapy.contrib.webservice":{enginestatus:[18,4,1,""],stats:[18,4,1,""],crawler:[18,4,1,""]},"scrapy.utils.trackref":{print_live_refs:[28,7,1,""],iter_all:[28,7,1,""],get_oldest:[28,7,1,""],object_ref:[28,5,1,""]},"scrapy.contrib.loader.processor":{MapCompose:[0,5,1,""],Join:[0,5,1,""],Compose:[0,5,1,""],TakeFirst:[0,5,1,""],Identity:[0,5,1,""]},"scrapy.contrib.downloadermiddleware.httpproxy":{HttpProxyMiddleware:[19,5,1,""]},"scrapy.exceptions":{IgnoreRequest:[22,8,1,""],DropItem:[22,8,1,""],NotSupported:[22,8,1,""],CloseSpider:[22,8,1,""],NotConfigured:[22,8,1,""]},"scrapy.contrib":{downloadermiddleware:[19,4,1,""],exporter:[9,4,1,""],memusage:[17,4,1,""],webservice:[18,4,1,""],closespider:[17,4,1,""],memdebug:[17,4,1,""],loader:[0,4,1,""],statsmailer:[17,4,1,""],spidermiddleware:[29,4,1,""],logstats:[17,4,1,""],linkextractors:[25,4,1,""],debug:[17,4,1,""],corestats:[17,4,1,""],spiders:[20,4,1,""]},"scrapy.contrib.downloadermiddleware":{redirect:[19,4,1,""],httpcompression:[19,4,1,""],cookies:[19,4,1,""],retry:[19,4,1,""],stats:[19,4,1,""],httpcache:[19,4,1,""],robotstxt:[19,4,1,""],DownloaderMiddleware:[19,5,1,""],httpauth:[19,4,1,""],downloadtimeout:[19,4,1,""],defaultheaders:[19,4,1,""],httpproxy:[19,4,1,""],useragent:[19,4,1,""],chunked:[19,4,1,""]},"scrapy.webservice.scrapy.webservice":{WebService:[17,5,1,""]},"scrapy.spider":{BaseSpider:[20,5,1,""]},"scrapy.contrib.logstats":{LogStats:[17,5,1,""]},"scrapy.contrib.closespider.scrapy.contrib.closespider":{CloseSpider:[17,5,1,""]},"scrapy.contrib.pipeline":{images:[13,4,1,""]},"scrapy.contrib.downloadermiddleware.httpauth":{HttpAuthMiddleware:[19,5,1,""]},"scrapy.item.Item":{fields:[8,9,1,""]},"scrapy.contrib.exporter":{JsonLinesItemExporter:[9,5,1,""],PprintItemExporter:[9,5,1,""],XmlItemExporter:[9,5,1,""],BaseItemExporter:[9,5,1,""],PickleItemExporter:[9,5,1,""],JsonItemExporter:[9,5,1,""],CsvItemExporter:[9,5,1,""]},"scrapy.http":{HtmlResponse:[26,5,1,""],Request:[26,5,1,""],XmlResponse:[26,5,1,""],TextResponse:[26,5,1,""],FormRequest:[26,5,1,""],Response:[26,5,1,""]},"scrapy.contrib.downloadermiddleware.stats":{DownloaderStats:[19,5,1,""]},"scrapy.contrib_exp":{djangoitem:[33,4,1,""]},"scrapy.spider.BaseSpider":{start_urls:[20,9,1,""],allowed_domains:[20,9,1,""],name:[20,9,1,""],parse:[20,6,1,""],make_requests_from_url:[20,6,1,""],start_requests:[20,6,1,""],log:[20,6,1,""]},"scrapy.contrib.statsmailer.scrapy.contrib.statsmailer":{StatsMailer:[17,5,1,""]},"scrapy.contrib.webservice.crawler":{CrawlerResource:[18,5,1,""]},"scrapy.contrib.exporter.BaseItemExporter":{encoding:[9,9,1,""],serialize_field:[9,6,1,""],export_empty_fields:[9,9,1,""],export_item:[9,6,1,""],finish_exporting:[9,6,1,""],start_exporting:[9,6,1,""],fields_to_export:[9,9,1,""]},"scrapy.contrib.downloadermiddleware.httpcache":{HttpCacheMiddleware:[19,5,1,""]},"scrapy.contrib.downloadermiddleware.DownloaderMiddleware":{process_response:[19,6,1,""],process_exception:[19,6,1,""],process_request:[19,6,1,""]},"scrapy.mail":{mail_sent:[31,7,1,""],send:[31,6,1,""]},"scrapy.contrib.webservice.stats":{StatsResource:[18,5,1,""]},"scrapy.http.FormRequest":{from_response:[26,11,1,""]},"scrapy.contrib.spiders.XMLFeedSpider":{process_results:[20,6,1,""],iterator:[20,9,1,""],adapt_response:[20,6,1,""],parse_node:[20,6,1,""],namespaces:[20,9,1,""],itertag:[20,9,1,""]},"scrapy.contrib.webservice.enginestatus":{EngineStatusResource:[18,5,1,""]},"scrapy.contrib.downloadermiddleware.chunked":{ChunkedTransferMiddleware:[19,5,1,""]},"scrapy.signals":{engine_started:[30,7,1,""],stats_spider_closing:[38,7,1,""],stats_spider_opened:[38,7,1,""],response_downloaded:[30,7,1,""],request_received:[30,7,1,""],stats_spider_closed:[38,7,1,""],item_scraped:[30,7,1,""],spider_error:[30,7,1,""],engine_stopped:[30,7,1,""],response_received:[30,7,1,""],spider_idle:[30,7,1,""],spider_opened:[30,7,1,""],item_dropped:[30,7,1,""],spider_closed:[30,7,1,""]},"scrapy.selector.XPathSelector":{register_namespace:[24,6,1,""],"__nonzero__":[24,6,1,""],extract:[24,6,1,""],select:[24,6,1,""],re:[24,6,1,""]},"scrapy.contrib.corestats":{CoreStats:[17,5,1,""]},"scrapy.contrib.downloadermiddleware.cookies":{CookiesMiddleware:[19,5,1,""]},"scrapy.extension":{ExtensionManager:[17,5,1,""]},"scrapy.contrib.spiders.CSVFeedSpider":{headers:[20,9,1,""],delimiter:[20,9,1,""],parse_row:[20,6,1,""]},"scrapy.contrib.loader":{ItemLoader:[0,5,1,""],processor:[0,4,1,""],XPathItemLoader:[0,5,1,""]},"scrapy.contrib.spidermiddleware.httperror":{HttpErrorMiddleware:[29,5,1,""]},"scrapy.contrib.pipeline.images":{ImagesPipeline:[13,5,1,""]}},terms:{serialize_field:9,canonicalize_url:25,name_out:0,prefix:[20,35,5,17,24],somenastyspid:28,oldest:28,whose:[29,19,13,4,21,26,8,20,16,25],aug:34,images_min_height:13,pprint:27,concret:[28,9],cmd_get_global_stat:18,under:[34,21,38,12],worth:0,everi:[29,2,19,20,35,25,36,17],risk:[16,28],telnetconsol:[5,17],spider2:[34,36],spider3:36,spider1:[34,36],personitem:33,affect:[13,21,5,28,9],trackref:[17,28],"__nonzero__":24,cmd:18,upload:36,reinvent:21,httpcache_ignore_schem:19,scrapybot:[34,5],images_thumb:13,"3afec3b4765f8f0a07b78f98c07b83f013567a0a":13,direct:[6,24,18,29],histor:14,second:[0,12,19,5,14,26,17],even:[10,29,0,28,13,20,21,14,30,33,34,25,26,17,39],get_stat:[38,18],neg:5,introspect:14,ptr:12,nonetyp:28,"new":[29,0,1,12,36,19,20,5,30,39,15,6,34,24,8,28,26,35,38,27,25],net:21,maverick:32,metadata:[26,0,8,19,9],customdownloadermiddlewar:19,avert:14,never:[1,28,19,5,26,17,39],here:[0,1,4,5,7,8,9,10,12,13,14,15,26,19,20,21,30,24,22,27,28,29,31,34,35,36,37],met:17,debugg:17,path:[29,1,3,13,4,5,19,15,7,24,37,17,18],process_result:20,interpret:[17,28],formnam:26,dropitem:[13,22,30,2],anymor:28,myextens:17,adopt:28,precis:[14,32],datetim:38,r156:36,sji:5,pending_request:27,aka:[6,19,36,9],dbmcachestorag:19,unix:[3,5,15,7,35,36],total:[28,14],unit:[15,0],highli:[35,3],get_media_request:13,describ:[29,0,3,20,5,23,19,30,6,21,24,36,26,9],would:[12,36,3,13,4,5,34,14,7,24,28,20,26,35,17,19,9],bleed:11,htmlpathselector:24,program:[6,12,27],call:[0,2,4,7,8,9,12,13,14,15,26,17,19,20,21,30,23,24,25,28,29,33,34,35,36,37,38],python25:3,python26:3,recommend:[12,3,20,21,35,37,18],xpath1:0,xpath2:0,type:[12,36,20,5,34,7,24,8,28,26,35,18,27,9],until:[29,0,28,19,13,6,26,16],get_oldest:28,relat:[33,26,35,37],yahoo:21,notic:[35,0,8,14,28],warn:[21,3,20,5,19,32,7,8,37,16,17,9],somepass:19,hold:12,must:[0,2,5,8,9,10,12,13,15,26,17,18,19,20,30,24,25,22,28,29,34,37,38],accid:28,anydbm:19,join:[15,20,0,14,9],err:36,setup:[36,3],work:[0,1,3,4,7,8,9,10,11,12,14,15,26,17,19,20,21,24,16,33,34,35,36,38,39],spidermiddlewar:[29,5],"0x1b6c2d0":12,storemyitem:5,root:[12,3,15,34,36,9],overrid:[0,19,13,20,5,34,33,21,36,26,9],defer:[30,21],gb18030:5,climat:14,give:[20,0],gotopag:25,smtp:31,indic:[22,20,26,19],caution:16,unavail:7,want:[0,2,4,9,10,11,12,13,14,15,26,18,19,20,21,24,27,28,29,35,36,38],memusage_report:[5,17],keep:[0,1,2,4,5,8,10,28,13,15,26,17,18,19,20,21,24,29,34,35,36,38,39],rid:11,feed_uri:7,end:[0,12,19,35,36,9],eng:14,travel:26,how:[0,1,2,3,4,5,6,9,10,11,12,14,15,26,17,18,19,20,21,30,24,27,28,29,31,34,35,36,37],env:18,answer:[11,28],collaps:15,recogn:26,chines:5,after:[12,29,0,1,2,36,38,4,5,31,14,30,21,24,28,20,26,35,17,9],downloadtimeout:[5,19],diagram:6,befor:[30,0,12,19,13,20,5,38,15,33,34,37,26,17,27,9],wrong:24,offlin:19,ovd:14,arch:3,parallel:[5,36],mimemultipart:31,mapcompos:0,third:28,classmethod:26,credenti:[7,36],exclud:[24,25],jobdir:10,receiv:[0,2,19,13,20,21,30,6,25,26,17,9],unpaus:27,maintain:0,environ:[11,5,28,3],enter:[12,28],exclus:19,first:[0,3,4,6,9,11,12,13,14,15,26,19,20,21,24,25,28,29,32,34,35,38,39],order:[10,29,0,12,19,13,20,5,21,35,25,36,26,17,27,9],dont_retri:[26,19],origin:[29,0,1,19,13,4,5,14,8,20,26],composit:0,feedback:12,atribut:33,xlib:[17,2,9],offici:[13,15,3],delv:34,becaus:[29,0,1,12,3,19,14,30,21,24,17],jpeg:13,stats_class:[5,38],flexibl:[0,9],vari:21,extract_unquot:24,fit:0,fix:[5,28,19,21,32,15,36,3],better:[15,0],drawback:24,enginestatu:18,persist:[10,20,30,6,38,9],hidden:26,easier:[15,4,1,12],descend:[4,2],closespider_itemcount:17,them:[0,1,2,3,4,5,6,9,10,11,12,13,14,15,26,17,19,20,21,30,24,16,28,29,33,34,36],thei:[0,2,4,5,6,7,8,28,13,14,15,26,17,19,20,21,22,24,25,12,32,34,35,38],fragment:28,safe:[10,20,39],oneir:32,jinja2:21,process_except:19,timeout:[17,19],each:[0,1,2,4,5,6,8,9,12,13,17,18,19,20,21,25,28,29,33,34,36,38],debug:[29,21,12,11,20,5,34,14,19,35,36,37,26,17,27,28],ws_name:18,mean:[11,0,12,36,13,20,5,30,7,24,21,26,39,28],httpproxi:[5,19],spider:[0,2,4,5,6,7,9,10,11,12,13,14,26,17,18,19,20,21,22,23,35,25,27,28,29,30,34,36,37,38],runtest:15,formrequest:[20,21,28,26],parse_length:0,ommit:[10,37],extract:[11,0,1,12,4,5,34,14,6,21,24,8,20,35,25],eggifi:36,network:6,webservic:[5,17,18],pyopenssl:3,facil:[10,11,31,14,37],content:[34,12,13,20,31,14,23,24,26,35],parse_page1:26,parse_page2:26,torrentitem:14,r99:36,got:[11,28],include_headers_lin:9,urllength_limit:[29,5],written:[21,12,5,15,6,7,28],nightmar:0,memorystatscollector:[5,38],free:[34,23,21,18,28],standard:[29,21,12,19,13,5,34,14,7,24,8,36,37,26,35,17,18],element:[0,1,12,13,4,24,20,26,9],breviti:[10,14],md5:13,items_count:10,depth_stats_verbos:5,traceback:[36,8,18],middlwar:[22,19],iso:5,unabl:[21,28],temporari:[35,36,19],user:[11,21,12,3,20,34,5,31,14,15,6,7,36,26,19],rang:[29,5,14,34,18,27],rank:4,system:[12,3,13,5,14,6,7,35,36,26,19,28],restrict:[36,5,8,14,28],hook:[6,19,27,14,29],instruct:[12,19,4,21,3,26],alreadi:[33,0,12,3,38,21,15,24,36,37,26,17,2],messag:[29,20,21,31,36,37,17],httperrormiddlewar:[29,5],uncollect:17,hood:12,tor:[21,14],top:[4,12],req_or_url:[35,12],sometim:[10,0,2,21,34,35,36,26,28],fiction:5,mercuri:36,cumul:28,master:32,too:[29,21,28,19,13,20,5,32,24,8,36],logged_in:20,john:[26,20,33,36,9],listen:[17,18,27,36],clientlanguage_nl:19,consol:[11,12,5,14,35,17,27,28],namespac:[20,5,27,24],tool:[11,1,28,4,5,34,35,36],setuptool:[36,28,3],djangoitem:[33,16],scraper:[7,27],scrapyd2:36,task:[0,12,4,5,30,6,24,8,38,27],def:[10,0,2,13,4,14,22,34,35,25,20,26,9,17,18,12],cdata:0,crawl:[10,11,21,12,36,29,4,5,34,14,30,6,19,35,20,26,17,28],target:[34,18,36],keyword:[26,20,0,9],provid:[0,5,6,7,8,9,10,11,12,13,14,26,17,18,19,20,21,30,24,25,28,31,33,34,35,36,37,38],project:[0,4,5,7,8,11,12,13,14,15,17,18,19,20,21,30,23,27,28,29,31,33,34,36],matter:[29,35,17,19],minut:28,httpauth:[5,19],spider_middlewares_bas:[29,5],runner:36,feed_storag:7,mind:[10,29,0,1,19,20,5,34,15,21,24,36,26,35,18,39],mine:[12,14],raw:[0,12,19],aforement:12,vat_factor:2,increment:[38,9],"__main__":18,seen:[13,0,14,19],myspid:[29,20,21,34,35,36],latter:[20,26],client:18,csviter:21,jsonrpc_client_cal:18,field_in:0,though:[0,12,20,34,14,30,7,26,17,28],usernam:[26,36],object:[0,2,4,5,8,9,12,26,17,18,19,20,30,24,25,27,28,29,31,35,36,37,38],what:[29,0,1,12,11,20,3,34,14,15,21,35,28,33,9],regular:[0,12,19,4,21,14,34,24,25,20,37,26,35,17,27],linkextractor:[4,25,20],tradit:[15,0],simplic:9,don:[10,29,0,2,19,13,4,5,30,34,15,21,36,25,20,17,12],dom:[4,21,1,12,20],doc:24,flow:6,doe:[29,21,1,3,38,19,30,24,8,26,17,9],declar:[0,12,20,5,14,8,26,9],yann:14,unchang:[0,9],dot:24,class_nam:28,"__str__":26,random:[20,5],syntax:[5,34,0,8,12],earth:14,involv:36,absolut:[7,24,25],acquir:0,delet:[1,2,36],explain:[15,4,24],response_head:19,apach:34,downloader_debug:5,folder:3,jsonresourc:18,googlegroup:36,nasti:28,stop:[10,30,0,19,21,22,35,38,18,27],amazon:[13,7,5,14],iso8859:5,xvid:14,report:[11,28,5,15,34,27],bat:15,bar:4,sgmllinkextractor:[4,25,14,20],excel:35,"public":[15,25,32],reload:17,bad:[29,24,20],unnecesari:20,ban:21,steal:21,runn:17,testitem:20,mandatori:[20,7,12],result:[29,0,13,4,24,20,9],respons:[0,4,5,6,10,11,12,14,26,17,19,20,21,30,24,25,22,28,29,34,35,36],fail:[13,26,19],"_out":0,best:[15,20,23,21],subject:[15,31],awar:[4,26],said:[33,12],redirect:[26,5,19],databas:[6,36,2,14,19],urllib:[18,19],discoveri:36,figur:[12,28],invalu:35,simplest:[21,12,20,5,0,36],irc:[15,11],approach:[4,24,39],attribut:[10,29,0,1,2,12,4,25,5,28,19,7,24,8,20,26,38,34,9],accord:[29,20,19],extend:[11,0,12,5,14,30,6,7,8,36,26,38,25],"72811f648e718090f041317756c03adb0ada46c7":19,extens:[10,11,1,12,19,38,5,30,31,14,22,23,25,36,26,17,18,27,28],clickdata:26,image1_thumb:24,protect:12,expos:18,howev:[29,0,12,36,4,28,38,24,20,26,35,17,27,9],against:12,logic:[29,20],countri:26,login:[20,21,26],browser:[5,1,12,19,4,21,34,35,26],com:[10,29,12,36,19,13,4,34,5,23,31,7,24,20,26,35,28],dont_merge_cooki:26,loader:[15,11,0,14],guid:[29,1,12,11,5,19,35,3,18],assum:[38,12,20,24,17,18,9],duplic:[10,26,5,2,25],uniod:26,pkgutil:36,fri:34,three:[0,12],been:[30,2,5,31,15,6,37,38,12],accumul:28,much:[29,0,12,33,24,8,35],allow_domain:25,interest:[26,34,12,14],subscrib:16,"__doc__":18,quickli:[10,11,0,12,21,24,35,38,27],arena:28,deeper:12,lifo:21,dismiss:15,telnet:[11,28,5,14,17,27],argument:[10,22,0,12,13,20,25,28,30,15,34,24,8,36,37,26,9],downloader_stat:[5,19],"catch":[30,14],googlesitemap_depth:17,cmd_list_avail:18,ident:[26,0],servic:[11,5,38,21,14,34,36,37,17,18],properti:[33,12],commerci:21,memdebug_notifi:5,aim:[20,38],spawn:36,after_post:26,have:[0,2,4,5,6,9,11,12,13,14,15,26,19,20,21,30,24,28,29,34,35,36,39],xpathselectorlist:24,tabl:[38,1],memdebug:[5,17],request_or_url:35,player:9,kwarg:[0,18,9],sumbit:26,conf:[5,27,36],tediou:[4,12],sever:[0,2,13,20,30,34,24,25,38,18,12],close_spid:[30,38,2,18],grown:28,perform:[29,12,19,20,5,31,14,30,24,8,28,26,17,2],suggest:15,make:[10,29,1,12,19,13,4,21,31,14,36,17,9],get_output_processor:0,complex:[13,4,12,29],split:[5,12,9],examplespid:28,complet:[30,12,19,13,15,24,35,39,28],fragil:26,blue:0,othersit:29,hand:1,rais:[29,2,19,13,38,21,30,22,26,17],parse_pag:22,kept:0,thu:[26,6,0,28,19],hypothet:[17,2,28],inherit:[20,0,38,28],contact:21,formnumb:26,thi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,33,34,35,36,37,38,39],start_export:9,gzip:19,programm:[24,12],everyth:11,left:[29,17,14,19],json_get:18,protocol:[19,18,9],just:[0,2,3,4,8,9,10,12,13,14,17,18,19,20,21,24,27,32,33,34,35,36,38],laptop:8,bandwidth:22,human:9,"__dict__":28,yet:[0,3,21,16,38,39],languag:[21,5,12,24,9],previous:[0,8,28],easi:[29,0,2,19,20,31,14,17,12],builtin:[29,21,24],els:[0,2,14],save:[33,12],plasma:0,xml_respons:24,func_glob:28,sanit:14,applic:[5,28,21,14,36,37,9,12],mayb:[4,20,3],field_nam:0,httprespons:30,ineffici:19,apart:20,measur:21,daemon:36,specif:[11,0,3,38,5,34,14,15,6,19,24,36,17,9],arbitrari:[26,0],manual:[30,4,3],noth:[20,38],spider_nam:18,genspid:[34,5],underli:[37,9],www:[10,29,12,3,13,20,5,34,14,19,35,26,28],right:[30,38],old:[21,8,28],deal:[29,0,21,14,24,36,26,17],interv:5,begin:[20,12,9],httpcach:[5,19],intern:[13,30,0,28],flatten:[0,24],interf:31,successfulli:[13,36],itertag:20,txt:[20,5,14,19],deploy:[37,14],"26d1b1a6d6f111e0be5c001e648c57f8":36,subclass:[33,0,12,20,25,5,30,24,8,28,26,18,9],tracker:[15,11],middlewar:[11,21,12,29,13,20,5,23,14,22,6,19,36,26,17,28],condit:17,foo:0,localhost:[28,31,36,38,18,27],core:[15,20,5,17,27],clear_stat:38,shift_ji:5,repositori:36,post:[26,11,35,36,20],"super":[20,9],chapter:0,item_scrap:30,py26:18,slightli:34,simul:[26,21],unsav:33,canonic:25,lastseen:27,commit:33,produc:[5,0,9],win32api:21,aws_secret:7,"float":5,guppi:28,bound:28,down:[21,5,28],wrap:18,opportun:20,byvia:28,accordingli:35,git:[23,36],cmdname:18,wai:[34,0,12,3,13,28,5,23,14,15,21,24,8,36,37,17,18,39,9],support:[10,30,0,3,13,20,5,34,14,32,22,7,36,8,21,26,19,9],"class":[0,1,2,4,5,6,7,8,9,11,12,13,14,15,26,17,18,19,20,21,24,25,28,29,31,33,34,35,38],avail:[0,5,7,8,11,12,26,17,18,19,20,23,24,25,27,28,34,35,36,37,38,39],width:[13,1],reli:[10,30,39],images_stor:13,editor:[34,5],overhead:29,head:[24,12],local_set:36,form:[26,21,38,9],altogeth:25,image5_thumb:24,nolog:[34,35],pagerank:4,heap:28,"true":[21,19,13,4,25,5,24,8,20,37,26,17,18,27,9],flavour:12,attr:25,bugfix:39,maximum:[29,5,17,36,19],maximun:5,tell:[5,15,0,14],r382:36,log_encod:[37,5],emit:31,iternod:20,featur:[11,1,12,13,4,38,14,15,7,39,36,16,17,22,9],"abstract":9,filedownloadhandl:5,exist:[1,17,8,36,26,38,39],glanc:[11,14],ship:19,redirect_max_metarefresh_delai:[5,19],check:[2,13,28,5,14,15,34,35,36,26,9],mulitpli:36,output_processor:0,image3_thumb:24,when:[0,1,2,4,5,6,7,9,10,11,12,13,14,15,26,17,19,20,21,30,24,25,28,29,33,34,35,36,37,38],elementtre:24,jone:28,test:[10,11,0,1,19,4,15,24,8,35],node:[20,24,12],offsit:[29,21,5],intend:[30,29,20,19],urllib2:19,intent:34,award:4,dbs_dir:36,xmliter:21,max_proc:36,longer:[10,29,2,20,15,9],home:[20,14],some_callback:10,exce:[5,17],ignor:[0,28,19,5,22,25,26,38,9],distro:[32,27,3],time:[10,29,0,28,19,20,5,14,38,7,35,25,36,26,17,27],backward:39,shop:20,mydomain:[34,36],concept:[11,12,21,14,24,8],rom:12,chain:[29,0],consum:[29,7,21],signific:[5,28],customari:17,per:[29,2,19,13,28,5,30,34,15,7,36,38,9],informit:12,row:[20,28],decid:[29,28,2,14,19],depend:[29,0,19,3,30,21,8,36,26,17],decim:5,readabl:[15,31,9],certainli:34,decis:9,aspx:12,insophia:32,sourc:[0,1,12,36,3,4,21,14,32,15,34,24,8,20],string:[0,12,36,4,5,31,30,34,24,25,20,26,17],yourspid:20,download_timeout:[5,19],marshalitemexport:7,word:[21,24,20,5,0,8],archiv:[11,14,32],administr:36,level:[5,12,19,20,21,37,26],did:[20,21],reproduc:15,leftov:9,dig:12,iter:[29,0,28,20,21,31,24,26,38],item:[0,2,4,5,6,7,8,9,10,11,12,13,14,15,26,17,18,20,21,22,35,28,29,30,33,34],unsupport:22,cooki:[10,5,1,28,19,21,14,26],div:[24,0,12,14],dir:[13,5,19],prevent:[30,20,33,21,12],crawlspid:[29,19,4,21,14,34,25,20],sign:32,histori:15,port:[5,21,31,36,17,18,27],myimagespipelin:13,appear:[37,4,5,31],currenc:26,beautifulsoup:[21,24],current:[0,1,28,19,13,38,5,34,35,36,37,17,27],webservice_resources_bas:18,response_bodi:19,slot:[27,36],"0x9e272ac":17,urlopen:18,gener:[29,12,19,13,20,28,31,14,32,30,6,7,36,26,17,9],slow:[21,24],address:[15,5,31],along:[26,5,1,31],wait:[30,34,5,28,19],box:[10,30,7],checksum:13,parse_opt:18,bot:[21,5,14],txrequest:18,proud:21,queue:[28,13,21,15,36,18,27],trial:15,behav:34,spider_clos:[28,17,30,9,38,2],overriden:[34,9],extrem:38,commonli:[20,7,0],ourselv:21,useful:33,extra:[33,8,36],modul:[10,11,21,12,19,20,5,15,34,36,25,28,37,38,17,27,9],prefer:[20,5,31],httpcache_ignore_miss:19,subcategori:4,visibl:15,instal:[11,12,3,13,21,32,7,35,36,17,27,28],regex:[20,0,24],fresher:32,memori:[11,21,28,20,5,38,17,18,27,9],sake:14,pref:[28,27],visit:[10,26],todai:[0,8,14],live:[4,1,28,14],handler:[5,19,21,30,26,27],ignored_extens:25,msg:[37,20,17,31],scope:5,checkout:23,plug:[6,14,29],peopl:28,finit:28,bot_nam:[34,5],redirectmiddlewar:[5,19],loginspid:26,easiest:3,uniqu:[20,25,36,2,12],cat:14,minimum:[13,15,5,37],can:[0,1,3,4,5,7,8,9,10,11,12,13,14,15,26,17,18,19,20,21,22,23,24,25,27,28,29,30,31,32,33,34,35,36,37,38],purpos:[2,4,25,21,28,14,34,35,8,20,17,18,9],problemat:21,stream:9,agent:[21,5,34,14,19],topic:[17,31],critic:[37,5],recycl:28,download_handl:5,unfortun:28,r23:36,occur:[15,6,30,11],alwai:[0,28,19,15,34,36,37,26,38,9],differenti:34,lxml:[20,21,24,3],multipl:[0,2,19,20,34,14,15,7,36,26,9],body_as_unicod:26,charset:34,write:[11,21,2,12,36,29,4,19,31,14,15,34,35,20,17,18,9],jsonlinesitemexport:[7,9],takefirst:0,actual:[0,1,4,14,24,20,37,35],csvitemexport:[7,9],tbodi:[4,1],name_in:0,map:[5,0],product:[11,0,12,13,20,21,14,24,8,28,37,35,39,9],jsonitemexport:[7,21,9],item_el:9,dive:12,spot:21,mac:3,mai:[0,2,3,4,7,9,10,12,14,15,26,17,19,20,21,30,24,16,28,29,34,35,36,37],log_level:[37,5],data:[0,1,2,4,6,7,8,9,10,11,12,14,26,18,19,20,21,24,25,33,35,36],grow:[0,39],subsequ:[29,21,12,19,13,20,5],gethostnam:38,quot:28,jsessionid:19,inform:[5,6,9,11,12,13,14,15,26,17,18,19,20,21,30,25,22,27,28,29,34,37],preced:[5,0,25],combin:20,callabl:[26,20,0,25,9],googledirectoryspid:4,allowed_domain:[29,12,4,14,34,20],talk:20,extractor:[11,25,20],delproject:36,nbsp:14,still:[29,0,28,19,21,38],mainli:20,process_respons:19,dynam:[5,18,27],equiv:26,snippet:[23,0],group:[0,28,12,6,25,2],replace_valu:0,polici:[15,26,21,5],platform:[15,34,36,17,3],window:[3,5,15,35,36,17,27],mail:[15,11,17,31,16],main:[18,32,8,36,12],non:[29,0,12,19,5,31,14,6,24,17],encod:[26,19,5,14,9],supersed:5,sooner:15,now:[20,28,4,24,36,38,12],discuss:15,nor:[0,28],introduct:[4,12],mailfrom:31,pyxi:12,name:[0,4,5,7,8,9,12,13,14,15,26,17,18,19,20,21,23,24,25,28,29,31,32,33,34,36,38],drop:[13,30,2,19],django_model:33,separ:[13,20,0,8,3],sigusr2:17,updat:[35,32,12,27,3],errback:[29,26],compil:[20,0,24],domain:[29,21,12,19,20,5,38,6,34,25,26,17],replac:[0,32,7,35,8,36,26],continu:[29,2,12,3,19,32,35,36,26,17,9],contrib:[29,0,19,13,4,5,15,7,25,20,17,18,9],make_requests_from_url:20,yield:[13,4,12,20],year:14,urlpars:18,happen:[5,0,12,28],shown:[12,4,5,6,35,26],space:28,storag:[11,19,13,5,14,7],feed_store_empti:7,rational:5,internet:19,instati:0,modelform:33,factori:0,export_item:9,earlier:12,image_info_or_failur:13,orm:12,crawlerresourc:18,parse_categori:[4,20],after_login:26,org:[12,3,20,14,32,24,35],"byte":[34,28],unpredict:24,care:[10,26,36],reusabl:14,depth_limit:[29,5],gb2312:5,turn:[6,17,28],place:[6,0,8,20],charfield:33,view:[1,12,34,35,27,28],think:21,frequent:[11,7,21,32,16],lambda:[10,0,25],oper:[1,28,36],directli:[0,1,21,14,37,26],onc:[28,19,13,4,21,6,35,25,20,26,9],yourself:[20,34],lsb_releas:32,submit:[15,26],webservice_en:18,open:[30,2,12,19,4,21,15,6,34,24,20,38,35,17,27,9],size:[13,28,14],given:[0,12,19,20,25,28,31,34,24,8,36,26,35,38,9],sheet:35,sigquit:17,convent:[15,36],checker:1,max_length:33,useragentmiddlewar:[5,19],mail_port:31,white:0,conveni:[11,0,28,13,20,21,6,34,35,8,26,38,27,9],filter:[10,29,21,2,19,13,20,5,14,25,26],copi:[26,21,8],specifi:[29,0,28,19,13,20,25,5,15,33,7,24,8,36,26,17,18,9],github:[15,23,12,32],enclos:0,necessarili:21,than:[29,0,38,19,13,4,21,32,15,20,17],serv:[21,25],wide:[1,14],newspider_modul:5,posix:17,were:[13,35,2,19],posit:0,max_proc_per_cpu:36,baseitemexport:9,seri:39,pre:[26,11,34,20],response_receiv:30,sai:[2,4,14,15,36,12],euccn:5,ani:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,16,17,18,19,20,21,22,23,24,25,26,28,29,30,31,33,34,35,36,37,38],get_collected_valu:0,dash:[0,39],sax:12,deliv:30,engin:[29,19,38,30,6,35,26,17,18,27],advic:1,alias:5,destroi:28,note:[10,0,2,13,5,32,30,7,24,8,36,35,17,12],take:[10,2,4,5,14,6,36,20,17,12],green:6,parmet:7,csvfeedspid:20,channel:[15,11],xmlpathselector:24,sure:[10,19,13,4,15,36,9],trace:17,normal:[13,17],track:[29,28,19,38,21,36,17],price:[9,0,2,24,8],stacktracedump:17,compress:[14,19],statu:[29,19,21,36,26,17,18,27],detect:[20,5,14],pair:[10,19],homepag:26,size_nam:13,downloadermiddlewar:[5,19],later:[10,0,12,6,36,26],statsmail:[5,17],max_valu:38,recipi:[17,31],serializ:10,show:[28,4,5,15,6,34,36,27],remove_cdata:0,jobitem:35,concurr:[6,5,28,36,19],permiss:1,hack:5,smtpport:31,help:[11,0,12,4,14,15,34,35,18,28],xml:[11,0,12,20,5,14,7,24,21,26,35,9],onli:[0,4,7,8,9,12,13,26,17,18,20,21,30,24,25,27,28,29,34,35,36,38,39],explicitli:[37,0,17],activ:[29,0,2,19,5,34,17,27,28],enough:[20,12],s_product:9,dict:[10,29,0,2,19,13,20,5,38,7,8,28,26,17,27,12],analyz:[21,5],over:[0,1,2,4,24,25,20,38,28],add_opt:18,min_valu:38,get:[0,3,4,5,6,7,8,10,11,12,13,14,15,26,17,18,19,20,21,30,24,16,28,29,32,33,34,35,36,37,38],repr:19,repo:32,extract_unoquot:24,requir:[10,0,12,36,3,20,5,34,19,15,7,24,28,21,26,9],item_nam:20,aptitud:[36,32],email:[11,5,17,31],scientist:14,http_pass:19,where:[29,0,12,36,19,13,20,34,5,31,14,30,7,35,25,21,38,17,28],main_url:26,htmlxpathselector:[12,24,4,14,35,20],proce:35,spider_modul:5,sean:12,"0xa05670c":17,item_descript:20,review:[15,14],enumer:24,label:26,behind:14,between:[10,6,5,14],"import":[0,2,4,5,8,9,10,12,13,14,17,18,20,21,35,27,28,31,32,34,36,37,38],across:26,parent:4,screen:[24,14],fundament:0,cycl:20,unpopul:9,cookies_en:[21,19],come:[29,0,1,12,19,4,36,24,20,18,27,28],sitemap_follow:20,ip_isocod:19,img:24,region:[0,25],myitem:[13,20,34,26],tutori:[4,23,12,14,11],start_url:[12,4,14,34,20,26],mani:[0,28,19,5,15,24,36,3,39],among:[35,0,1,24,9],undocu:13,color:[35,0,9],overview:[6,38,28,11],inspir:21,pop:9,default_loader_context:0,exploit:17,jsonrpcerror:18,cancel:[22,30],poll:36,moduledict:28,coupl:[26,24,12],ms936:5,object_ref:28,breath:21,addit:[30,0,12,19,13,25,14,15,6,7,24,8,36,26,35,33,9],repons:[26,29,21,36,19],resolut:26,catastroph:14,those:[0,2,4,5,8,9,12,13,14,26,17,19,20,21,30,24,25,22,28,29,31,32,34,35,38,39],sound:26,bot_vers:5,invok:[35,21,17],mail_host:31,stdout:7,metric:[18,14],worri:12,memusag:[5,17],ascii:[26,5,12],twistd:36,"__init__":[12,34,36,9,17,18,2],http_port:36,develop:[1,28,15,34,35,36,37,39],author:[15,12],alphabet:[5,36],same:[10,34,0,2,36,19,13,4,5,38,28,15,7,24,8,20,26,35,17,39,12],binari:26,html:[29,0,1,2,19,4,5,34,14,21,24,25,20,26,35,12],pai:12,shell:[11,5,12,4,21,14,34,24,35,27],document:[29,21,12,3,4,5,30,28,14,26,15,6,19,24,11,8,20,16,39,9],finish:[2,19,13,38,5,14,30,6,17,12],listproject:36,screenshot:4,nest:[20,24,12],someon:31,driven:6,capabl:26,someintranetsitespid:19,improv:[15,18,28],extern:[7,35],download_handlers_bas:5,"break":39,appropri:19,megabyt:[5,28,14],markup:[4,0,24,20],join_multivalu:9,addvers:36,without:[21,28,24,19,5,31,30,34,0,26,35],model:[33,28,35,8,12],comment:15,execut:[29,0,1,2,19,26,17,18,27,12],tip:[21,1],rest:[11,21,31,28,19],depth_prior:[29,21,5],kill:17,stats_en:[5,38],http_proxi:19,speed:[21,5,24],aws_access_key_id:[7,5],trigger:[6,17],except:[29,0,2,11,13,20,19,30,23,14,22,21,8,36,26,38,17,18,9],desktop:8,blog:15,textrespons:[26,5],real:[21,12,24,28],hover:4,around:[21,18],delvers:36,read:[0,28,14,15,26,18,9],mimenonmultipart:31,traffic:19,dispatch:[30,36,17,2,9],smtphost:31,world:[15,0],default_output_processor:0,mininova:14,response_download:[30,26],integ:[26,5,17,14],server:[5,19,21,34,7,24,36,17,18],process_valu:25,either:[29,0,2,19,13,20,24,25],"_static":24,output:[29,0,12,36,11,13,28,5,34,14,15,6,7,24,21,37,35,17,18,19,9],manag:[11,1,28,21,34,36,17,18,27],webservice_host:18,httpcachemiddlewar:[5,19],feedexport:[7,5],item_pipelin:[13,5,2],jsonrpcresourc:18,w3lib:3,mail_us:31,refresh:[5,19],httpcache_dir:19,process_spider_except:29,faster:19,confirm:34,notconfigur:[22,17],definit:[33,29,0,8,28],achiev:[35,24],exit:[35,17,18],extract_link:25,complic:12,formdata:[20,26],refer:[2,4,5,8,9,11,12,14,17,19,20,21,30,24,25,22,28,29,31,34,35,36],documentari:14,print_help:18,arrow:6,power:[28,35,12,14,9],garbag:[17,28],inspect:[11,1,12,4,21,35,27,28],renderget:18,broken:14,found:[12,19,20,5,14,24,28,26,35,9,2],image3:24,image2:24,referr:12,appli:[29,0,1,12,19,20,5,34,21,24,25,9],image5:24,image4:24,src:24,deflat:19,webservice_extens:17,degre:21,item_complet:13,processor:[5,0],effici:[11,0,38,14,19],consid:[19,21,15,25,36,9],cookiesmiddlewar:[26,21,5,19],strip:0,referermiddlewar:[29,5],your:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,30,35,25,26,27,28,29,31,32,33,34,36,37,38],load_item:0,compon:[29,12,19,20,5,22,6,8,36,26,2],fast:[20,12],log:[29,21,12,11,13,20,5,34,14,19,36,37,26,17,18],area:25,aren:[33,29],length_out:0,image2_thumb:24,start:[10,0,12,20,14,30,34,24,36,37,35,38,39],compliant:17,interfac:[3,24,25,36,18,27],low:19,lot:[0,1,12,21,14,28],resum:[10,11,35,27],besid:[21,33,24,36,38,39],media:[26,14],programmat:14,verbatim:5,outdat:16,tupl:[13,20,31,28,26],bundl:[20,24,25,26,18,9],jul:34,httpproxymiddlewar:[21,5,19],statsmailer_rcpt:[5,17],pull:[15,5],mertz:12,"default":[0,5,7,9,12,13,26,17,18,19,20,21,30,24,25,27,29,31,33,34,35,36,38],pacman:3,discountedproduct:8,basespid:[29,2,19,20,28,30,34,35,37,26,38,12],connect:[2,19,13,31,34,17,18,27,9],gone:30,cp819:5,creat:[34,33,0,1,12,36,19,13,4,25,5,28,15,7,35,8,20,26,21],certain:[11,2,19,20,21,30,6,34,24,8,36,35,17,18],site_id:7,my_seri:8,deep:28,decreas:[5,28],firebug:[4,1,12,11],file:[11,21,12,19,13,20,34,5,23,14,31,15,7,35,36,37,26,9,18,2],price_in:0,nerd:35,film:14,fill:[26,29,20],jsonlin:7,again:[26,12,19],googl:[4,21,17,24],collector:[28,17,5,38,18,27],get_data:36,runspid:[34,21],field:[29,0,2,13,12,14,33,8,20,26,9],valid:[2,13,5,33,6,26],collis:17,you:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15,26,17,18,19,20,21,30,23,24,25,27,28,29,32,33,34,35,36,37,38],architectur:[6,28,11],gettabl:32,file_object:31,docstr:15,latin_1:5,picklefifodiskqueu:21,pool:[21,28],reduc:[18,28],csvfeed:34,directori:[10,12,3,13,4,5,34,19,15,7,36],descript:[12,20,34,14,6,23,4,18,27],set_stat:38,tricki:21,goe:[29,0,19,13,20,6],potenti:19,escap:26,cpu:36,unset:[18,36],represent:26,all:[0,2,3,5,6,8,9,10,11,12,13,14,15,26,17,19,20,21,22,24,25,28,29,30,34,35,36,37,38],logs_to_keep:36,illustr:[0,12,15,7,24,8,36,38,18],images_min_width:13,follow:[0,1,2,3,4,5,6,7,10,11,12,13,14,15,26,17,19,20,21,24,25,28,29,32,36],disk:[10,36],children:24,set_trac:21,dmoz_spid:12,validatemyitem:5,request_fingerprint:5,introduc:[2,14],"case":[0,12,36,4,5,34,14,15,21,24,20,26,38,28],liter:[0,27],r10:36,fals:[0,19,13,20,25,5,33,7,24,8,37,26,17,27,9],faq:[15,11],multi:[26,9],robotstxtmiddlewar:[5,19],util:[21,28,19,5,15,0,25,18],mechan:[29,0,12,20,5,30,14,15,6,21,24,26,17,28],get_command:18,veri:[38,0,1,12,36,4,25,28,31,14,15,24,8,20,35,17,39,9],ticket:15,render_get:18,althought:18,occurr:0,list:[0,1,2,5,8,9,11,12,13,14,15,26,18,19,20,21,22,24,25,16,28,29,30,31,32,34,35,36],adjust:[13,5,2],stderr:18,scrapy_settings_modul:5,small:[13,15,24,12],get_engine_statu:18,dimens:13,context:0,process_request:[20,19],arthu:14,handi:[1,12],zero:[29,5,17,39,19],design:[5,20,0,14],pass:[10,30,0,12,19,13,4,34,28,31,15,6,7,36,25,20,37,26,17,18,9],further:[30,0,2,12],val1:36,newfaq:5,proxi:[21,19],deleg:9,sub:[28,13,14,15,34,36],section:[0,12,5,6,34,24,36,16,38],spideropencloselog:17,abl:[10,29,0,1,28,4,6,7],brief:6,overload:19,spider_idl:30,version:[11,0,3,5,34,15,21,36,26,39,9],consecut:5,method:[0,2,4,5,9,12,13,15,26,17,18,19,20,30,24,25,27,29,33,34,35,36,37,38,39],price_excludes_vat:2,hasn:39,full:[1,12,13,15,36,17],hash:13,unmodifi:9,sophist:21,behaviour:[0,13,20,5,34,21,8,26,18,27],shouldn:[36,17,18,25],intranet:19,strong:14,modifi:[21,1,20,5,34,0,25,35],valu:[0,5,7,8,9,10,12,13,14,26,17,18,19,20,24,25,29,31,34,36,37,38],adapt_respons:20,search:[11,4,5,15,25,36],sender:31,robotstxt:[5,19],ftpfeedstorag:7,getbool:5,noitem:34,prior:38,amount:[15,5,17,9],pick:[29,36,19,14,3],action:[29,2,19,4,6,26],via:[26,28,31],httpcompress:[5,19],httpcache_en:[17,19],basesgmllinkextractor:25,filenam:12,href:[1,12,4,14,24,25,20],parsed_length:0,select:[0,12,4,5,14,24,25,20,26,35,38],ctrl:[10,30,35,17],regist:[20,24],two:[0,12,3,5,19,34,24,25,36,17,18,9],taken:[13,19,9],dmoz:[23,12],kick:[29,19],minor:28,more:[0,1,3,4,5,6,8,9,11,12,13,14,15,26,17,18,19,20,21,22,23,24,25,27,28,29,30,32,34,35,36,38],cookies_debug:[21,19],desir:[10,26,12],hurri:9,flag:26,stick:19,particular:[29,0,1,28,19,13,20,5,38,34,37,26,17,9],known:[5,1,28,19,21,15,34,35,18],set_valu:38,downloader_middlewar:[5,19],cach:[5,28,19,21,14,26,17],none:[29,0,12,36,19,13,20,5,31,33,7,35,25,28,37,26,38,18,27,9],aws_secret_access_kei:[7,5],fa4dc0c496c8762ae4f1a620eab34f38:19,outlin:6,jobid:36,memorydebugg:[5,17],remain:[13,22,0,17,26],tpip:12,del:2,sent:[29,21,1,2,19,13,38,5,31,30,37,26,17,18,27],caveat:[4,1,36],learn:[11,23,12],deb:32,male:0,print_live_ref:28,scan:25,share:[10,0,13,4,14,15,23,24],accept:[30,21,5,15,34,0,8],mailsend:31,projet:36,xmlxpathselector:[20,35,24,12],get_output_valu:0,huge:21,cours:[4,34,28],goal:[8,39],divid:4,rather:28,anoth:[29,0,1,28,3,13,4,31,24,20,38],comfort:[21,3],open_spid:[38,2,18],from_respons:[26,21],opt:18,csv:[11,20,21,14,7,9],simpl:[11,2,12,38,25,31,24,8,36,17,18,9],mininovaspid:14,jsonrpc_cal:18,resourc:[29,2,19,28,14,30,36,38,18,12],referenc:28,variant:12,associ:[11,34,0,12,14],welcom:35,stabil:[11,39],"short":[24,28],footer:9,filesystemcachestorag:19,closespider_errorcount:17,examplifi:13,logstdout:37,django:[33,21,8],caus:[30,26,17,28,19],callback:[10,30,12,19,20,21,22,34,25,26,28],link_extractor:20,egg:36,isleaf:18,rotat:21,soon:[20,0],paper:28,through:[0,2,3,4,5,6,7,8,9,10,12,14,26,17,18,19,20,21,25,29,34,36,37,38],hierarchi:0,cp936:5,paramet:[29,0,2,36,19,20,5,30,28,31,22,7,24,25,21,37,26,38,27,9],style:[15,35],get_valu:[0,38],get_target:18,pend:21,reschedul:19,might:12,alter:19,request_receiv:30,good:15,"return":[0,2,5,6,8,9,10,12,13,14,26,18,19,20,21,30,24,25,28,29,36,38],timestamp:[7,36],framework:[29,28,19,20,21,31,14,15,6,36,17],deny_domain:25,request_with_cooki:26,bandwidth_exceed:22,placehold:12,bigger:[0,24],eventu:25,jsonencod:9,unlik:[26,24,9],authent:[19,21,31,14,36,26],easili:[11,21,14],token:26,returnth:12,fulli:12,unicod:[0,28,24,26,9,12],dnscache_en:5,hard:[26,5,28],idea:[15,29,21,12,14],procedur:21,realli:[29,1,2],xmlexportpipelin:9,concurrent_item:5,expect:[15,4,34,35,12],thing:[10,29,0,1,12,14,34,35,36,37,39],event:[30,6,11],ftp:[7,14],click_request_data:26,publish:32,etag:34,print:[10,29,12,36,20,5,34,24,8,28,37,35,17,18,27,9],sitemap_shop:20,asp:21,concurrent_requests_per_domain:5,advanc:28,xpather:1,suspici:28,effect:[5,28,9],quick:[37,31],reason:[30,0,28,13,20,5,38,22,23,24,8,36,26,17],base:[29,1,12,36,19,4,25,5,14,30,34,24,8,20,26,35,9],believ:21,ask:[15,6,21,12,11],mail_sent:31,basi:29,thrown:21,product_id:28,launch:[35,36],parse_product:20,omit:[37,20,7,31,19],lifetim:28,assign:[29,0,12,19,5,14,8,26,18,27],logstat:[5,17],feed:[11,21,2,12,5,14,6,7,24,20,9],singleton:[38,5,17],notifi:[30,16,5],obviou:14,upper:0,feel:[23,21,18,3],misc:5,number:[19,13,5,14,24,36,26,17,39],evolut:14,done:[21,14,3],construct:[0,1,4,5,14,24,20,35],stabl:[16,5,18,39,32],miss:2,stage:[13,22,30],differ:[10,29,0,12,36,11,4,25,5,28,19,34,24,8,20,26,18,9],script:[23,18,3],distutil:36,interact:[6,35,24,14,11],gpg:32,"573c1":34,least:[0,17],customspidermiddlewar:29,dfo:21,statement:14,natur:14,scheme:[7,19],store:[10,11,0,2,19,13,20,21,14,6,7,36,26,17,12],schema:20,option:[0,3,20,5,15,34,8,36,37,18],selector:[11,0,12,4,21,14,24,20,28],part:[24,28],pars:[10,0,12,36,20,21,14,6,34,24,28,26,35,9],consult:38,kind:[20,21,34,24,8,35,18],grep:19,remov:[36,35,0,27,19],reus:[13,0],str:[30,0,28,20,25,31,22,8,37,26,38,18,9],etlnet:27,initi:[26,20,17,8,12],grei:4,someothersit:29,comput:[11,35,12],add_valu:[15,0],unnotic:38,log_en:[37,5],packag:[15,11,32,36,3],expir:[10,13,5,19],"null":0,parse_item:[10,20,34],telnet_var:27,built:[0,5,7,8,9,10,11,12,14,26,17,18,19,20,21,30,24,25,22,27,28,29,32,34,36],equival:[26,0],start_request:20,self:[10,0,2,13,4,28,14,22,34,35,20,26,9,17,18,12],undeclar:12,also:[0,1,2,3,4,5,6,7,8,9,12,13,14,15,26,17,18,19,20,21,23,24,25,27,28,29,31,34,35,36,39],build:[21,12,36],pipelin:[11,2,29,13,12,5,30,34,14,22,6,19,28,20,17,9],distribut:[15,36,3],english:21,reach:[29,13,5,26,17,39],most:[10,29,0,12,36,11,20,34,5,28,7,24,8,21,26,18,27,9],plai:[11,23,35,12,36],plan:[21,28,39,36],prentic:12,user_ag:[34,5,19],charg:[20,12],listvers:36,filesystem:[7,14,19],field_out:0,clear:38,cover:[29,21,12,9],b0582849179d1de7bd86eaa7201ea3cda4b5651f:36,clean:[1,14],stats_spider_clos:38,dmozspid:12,usual:[5,29,0,17,26],optionpars:18,dont_click:26,session:[26,35,5,14,19],font:4,find:[11,12,4,21,32,15,28],impact:28,indexerror:18,pretti:[20,9],writer:9,bfo:21,eggs_dir:36,templat:[34,5,36],http_user:19,dont_redirect:[26,19],smtplib:31,httpcache_expiration_sec:19,urljoin:18,hit:[30,21,5,35],"__file__":36,express:[0,1,12,4,14,24,25,20,35],nativ:21,images_expir:13,longest:28,firecooki:1,is_idl:27,closespid:[22,21,5,17],attrbitu:25,item_id:20,statist:[11,5,17],set:[2,5,7,8,9,10,11,12,13,26,17,18,19,20,21,35,25,27,28,29,31,34,36,37,38],art:4,dump:[38,21,5,17,2],memoryusag:[5,17],startup:[38,17],sex:[33,0],see:[0,1,3,4,5,6,7,8,9,11,12,13,14,15,26,17,18,19,20,21,22,24,25,27,28,29,30,33,34,35,36,37,38],signifi:39,arg:[34,18,24,8,9],reserv:30,someth:[19,13,20,21,34,35,36],xmlproductload:0,won:[10,29,1,28,19,13,20,15,35,25,36,38,17],httperror:[29,5],nope:35,spiderst:5,commands_modul:34,altern:15,latin:5,appreci:36,javascript:[26,1,25],image_id:13,concurrent_requests_per_ip:5,solv:[11,21,28,3],classnam:34,popul:[29,0,12,11,13,20,5,33,8,37,26,17,9],both:[0,12,20,5,24,37,26,35,38,28],cryptic:21,last:[29,0,19,20,34,35,8,36,26,38],delimit:20,boto:7,ignorerequest:[22,19],retent:13,wed:34,foreign:[27,14],redirect_url:[26,19],"2b00042f7481c7b056c4b410d28f33cf":13,whole:[20,24,28,9],pdb:21,load:[10,12,20,5,24,36,17,18],simpli:33,point:[4,35,24,20],instanti:[0,12,20,5,24,25,17,9],schedul:[10,30,12,19,13,5,22,6,34,39,36,26,27,28],header:[21,1,12,19,20,5,31,34,28,26,9],shutdown:[30,5],linux:[17,27,3],encoding_alias:5,throughout:28,backend:[7,5,14,19],unsuccess:29,spidercontext:5,bertrand:14,due:[5,28,39],empti:[29,0,19,5,34,7,25,26,9],secret:[20,5,36,26],newcom:[15,28],urllength:[29,5],filtr:29,addison:12,register_namespac:[20,24],fire:30,imag:[11,1,13,14,24,26,35],great:21,deny_extens:25,understand:[15,11,34],func:18,look:[12,11,0,1,2,4,5,28,14,34,24,25,20,26,17,9],theref:4,erron:29,batch:10,s3feedstorag:7,"while":[0,1,28,19,5,35,36,26,17],smart:35,abov:[28,3,13,21,14,24,8,36,35],error:[19,13,5,14,30,35,36,37,26,17],fun:4,closespider_pagecount:17,loop:26,subsect:20,propag:26,readi:[15,34,24,14,36],readm:23,karmic:32,jpg:[13,24],itself:[4,21,27,20],cento:34,scraped_data:14,mac_roman:5,seem:[4,12],corestat:[5,17],irrelev:17,minim:36,belong:[29,28,12,20],decod:26,zope:3,inc_valu:38,memusage_notify_mail:[5,17],higher:[13,15,21],some_pag:26,mybot:[34,5,35],keyr:32,moment:[7,21],cp1251:5,dummystatscollector:38,cmd_help:18,cp1252:5,robust:14,getfloat:5,stack:17,telnetconsole_host:27,recent:[13,8],lower:[5,38],sha1:[13,36],lib:36,html_respons:24,older:[19,3],find_packag:36,productxmlexport:9,pickl:[10,7,19,9],person:33,deplet:14,macintosh:5,spend:38,downlod:34,propos:28,test_contrib_load:15,pywin32:[21,3],obscur:28,regardless:[15,26,38],project1:36,project2:36,password:[26,7,31,36],min_free_memory_perc:38,shortcut:[12,34,35,26,27,28],rgb:13,telnetconsole_port:[5,17,27],"_sre":28,input:[26,6,0,11],spider_stat:38,bin:[15,18],attach_nam:31,format:[11,0,2,19,13,12,5,14,7,9],big:[10,28,13,20,21,15],fifomemoryqueu:21,httpcompressionmiddlewar:[5,19],success:[13,29,12,20],signal:[10,11,2,19,38,21,31,14,30,26,17,27,9],root_el:9,resolv:[26,14],collect:[29,0,38,11,20,5,14,19,8,17,9],api:[11,0,12,5,31,14,15,34,24,8,36,16,38,18,39],popular:[6,24],filter_pric:0,sitemap:[20,17,14],often:[0,28,21,15,7,26,9],simplifi:38,some:[0,1,4,5,7,8,9,10,11,12,13,14,15,26,17,18,19,20,21,22,24,27,28,29,30,31,33,34,35,36,38],back:[26,29,21,12,19],global:[5,19,21,34,38,18],pprintitemexport:9,sight:4,scale:9,pep:15,other_callback:10,retri:[26,5,19],larg:[11,28,21,7,39,9],gnosi:12,prog:18,sample1:24,proc:0,previou:[29,0,19,20,21,8,38],run:[10,11,12,3,38,21,14,32,15,34,35,36,17,18,27,28],offsitemiddlewar:[29,21,5,20],spider_fil:34,step:[6,36,12,14,11],wget:5,major:39,extensionmanag:17,impos:[5,29,0,28],jsonrpc:18,constraint:13,optpars:18,idl:[30,34,5],block:[6,35,5,31],"0x2dc2b10":27,contributor:15,chang:[0,28,36,19,20,5,14,15,21,35,8,26,37,16,18,39],https_proxi:19,inclus:15,pages_crawl:38,authetn:31,enabl:[10,29,21,12,19,13,20,5,38,7,36,28,26,17,18,27,9],question:[15,11,21,28],parse_shop:20,custom:[11,21,12,29,13,20,5,6,34,36,25,28,26,17,9],includ:[22,1,2,19,13,20,25,5,38,39,15,6,24,8,36,26,17,27,9],suit:[20,25,9],parse_arg:18,xpath:[11,0,1,12,4,21,14,24,25,20,35],xpathitemload:0,properli:[16,7,21,28,35],"__viewstat":21,replace_xpath:0,link:[11,12,4,14,6,34,24,25,20,35],depthmiddlewar:[29,5],line:[11,12,28,5,34,32,15,7,24,36,37,26,35,9,38,2],info:[29,0,12,19,13,20,5,38,34,14,15,21,24,36,37,26,35,17,28],concaten:[0,12],utf:[26,34,5,9],consist:28,image4_thumb:24,getint:5,someus:19,similar:[29,0,12,20,5,34,24,8,35,17,28],inspect_respons:35,parser:[24,9],doesn:[10,29,0,28,19,13,20,5,14,38,35,8,36,26,17,39,9],repres:[26,11,17,12,20],"char":[15,9],incomplet:16,guarante:[13,26],curl:[36,32],scrapi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,34,35,36,37,38,39],itemload:[15,0],titl:[20,0,12,24],sequenti:2,scrape:[0,1,2,4,5,6,7,8,9,11,12,13,14,26,17,19,20,21,30,24,29,34,35,38],extensions_bas:[5,17],priori:28,s3downloadhandl:5,deseri:19,tmp:7,lucid:32,evan:28,randomize_download_delai:5,filefeedstorag:7,ago:28,algorithm:36,appen:0,mimetyp:31,dvd:9,beautifusoup:20,depth:[29,21,5,14],memusage_limit_mb:[5,17],far:[4,0,12,39,28],hello:[5,6,0,37],pluggabl:36,code:[0,1,3,4,5,6,9,10,11,12,14,15,26,17,19,21,23,24,25,28,29,31,34,35,36,38],edg:11,last_upd:[0,8],compact:28,pydispatch:[30,17,2,9],privat:39,friendli:[19,9],send:[10,11,21,19,5,31,15,6,36,26,17],update_telnet_var:27,another_subcategori:4,fields_to_export:9,whichev:26,get_input_processor:0,spider_error:30,mous:[4,1],spoof:14,relev:15,tri:26,depth_stat:[29,5],my_spid:21,"try":[11,12,19,21,14,32,15,35,36,26,18],resource1:18,feed_export:7,pleas:[15,32],default_set:5,somespid:[10,36],smaller:15,cfg:[34,12,36],otherproject:36,focu:21,possibl:[7,0,39,36,21],xmlfeed:34,download:[11,21,12,3,13,20,34,5,30,23,14,32,22,6,19,35,25,26,17,27],correspond:[12,19],odd:39,click:[26,23,12],append:[0,8,12],compat:39,index:[11,20,24,28,19],compar:[21,28,36],resembl:15,access:[0,12,19,38,5,14,8,36,17,18,27,28],experiment:[11,16],product2:[13,8],product1:13,"_spider_nam":18,whatev:[20,38],len:27,bodi:[12,19,4,5,31,22,24,20,26,35],let:[0,2,20,28,14,33,34,24,17,12],ubuntu:[11,32,36,3],parse_torr:14,becom:[16,0,12],redirect_max_tim:[5,19],sinc:[0,1,20,21,32,24,26,16,35,9],retrymiddlewar:[5,19],convert:[13,20,0,26],logs_dir:36,defaulthead:[5,19],fetch:[5,12,21,6,34,35],"48c9c87349680":34,other_url:[20,26],typic:[30,0,2,36,13,4,5,28,15,6,34,35,8,20,26,17,9],chanc:[15,5,28],configur:[29,12,11,25,5,31,19,7,8,36,37,17,9],firefox:[4,5,1,12,11],apr:19,win:5,app:36,foundat:21,apt:[32,3],"boolean":[13,20,5,24,25,37,26,17,18],"__name__":18,image1:24,input_processor:0,fed:[26,12],usd:26,from:[0,2,3,4,5,6,8,9,10,11,12,13,14,15,26,17,18,19,20,21,22,24,25,27,28,29,30,31,32,33,34,35,36,37,38],zip:23,commun:[14,36],next:[12,19,14,6,34,26,9],websit:[11,12,4,21,14,24,20],few:[10,28,20,21,31,24],log_stdout:[37,5],usr:18,en_en:19,panel:3,sort:[29,18,19],clever:1,"0x2bed9d0":35,appropi:26,robotstxt_obei:[5,19],getchild:18,gayotcom:27,clientform:26,concurrent_request:5,alia:[8,28],iter_al:28,some_xpath_selector:0,meet:[13,3],scatter:28,aliv:[17,28],control:[11,3,14,6,34,36,26,18,27,9],tar:3,memdebug_en:[5,17],process:[0,1,2,4,5,6,7,9,11,12,13,14,26,17,18,19,20,21,22,24,27,28,29,30,35,36],lock:13,sudo:32,tag:[19,4,14,24,25,20,26,35],tac:36,tarbal:[23,3],serial:[10,2,12,7,8,9],delai:[13,21,5,19],surfac:14,sit:6,tamper:1,httpdownloadhandl:5,subdirectori:19,instead:[0,28,19,20,25,5,34,21,35,8,36,37,26,38,18,9],scheduler_memory_queu:21,stock:[0,8],overridden:[29,0,19,20,5,34,26],httpcache_ignore_http_cod:19,constructor:[0,25,31,22,8,26,17,18,9],iter_spider_stat:38,attent:12,codetyp:28,aws_kei:7,retry_complet:19,alloc:[28,36],loglevel:37,light:19,encoding_aliases_bas:5,finish_export:9,issu:[10,11,28,19,21,15,26],mainten:0,allow:[29,21,1,12,36,19,13,4,5,14,7,35,25,20,17],movi:14,move:[1,28],add_xpath:0,comma:20,pricepipelin:2,mcgrath:12,directoryitem:4,chosen:20,euc:5,clickabl:26,stdoutfeedstorag:7,infrastructur:[21,5],request_bodi:19,strip_dash:0,therefor:[17,1],crash:21,greater:[38,36],python:[0,2,3,5,6,8,11,28,13,14,17,18,19,21,24,27,12,29,31,34,35,36],auto:[26,35,14],dai:13,auth:19,mention:[12,28],fingerprint:[5,19],xhtml:5,typo:12,anyth:[36,0,38,27,19],edit:[34,5,12,36],persit:10,retry_tim:19,mode:[13,5,36,9],notif:[11,17,31],feed_exporters_bas:7,subset:5,themselv:5,chunk:[5,19,9],getlist:5,meta:[10,29,28,19,5,26],"static":[11,35],redirect_en:19,our:[11,12,4,14,15,2],patch:[15,28],special:[20,21,0,35,26],out:[10,29,21,12,19,13,4,5,14,30,7,36,20,28],variabl:[12,19,5,24,3,27],product_titl:0,identifi:[4,1,12],categori:[4,34,14,20],rel:[1,12,19,13,5,31,24,28],default_item_class:[5,0],squeue:21,clone:26,common:[10,11,0,28,19,13,20,34,24,8,36,38,9],clarifi:0,shut:5,jsonwriterpipelin:2,insid:[29,0,12,36,4,5,14,6,34,24,25,28,17,27,9],workflow:13,csiso58gb231280:5,dictionari:[13,26,0,17,8],tempt:24,releas:[28,3,32,30,16,39],afterward:0,redirect_priority_adjust:5,default_input_processor:0,could:[29,12,19,4,5,14,35,8,20,26,39,28],put:[13,15,12,26],sitespecificload:0,david:12,length:[5,29,34,0],outsid:[29,34,21,17,36],retain:13,softwar:[21,12],dirbot:[23,12],isbn:12,suffix:0,serialize_pric:9,date:[34,39],specificproduct:8,statscollector:[5,38],prioriti:[13,26,5,36],"long":28,zxf:3,unknown:[26,8,18],natti:32,wrapper:24,attach:[13,31],termin:17,"final":[29,0,12,36,19,4,28,14,30,35,20,26,9],request_head:19,"63bbfea82b8880ed33cdb762aa11fab722a90a24":13,dupefilter_class:5,gotcha:10,shallow:26,rfpdupefilt:5,enqueu:6,exactli:15,haven:3,shelp:[35,12],cmd_list_run:18,bother:[29,19],structur:[12,19,14,34,24,8,36],charact:[20,5],mail_pass:31,sec:5,torrent:14,start_tim:[38,27],loader_context:0,default_response_encod:5,mybucket:7,clearli:4,robot:[20,5,14,19],myproject:[29,0,2,19,20,5,34,36,17],spiders_dev:5,notsupport:22,close:[30,12,38,5,22,6,34,9,17,27,2],need:[0,1,2,3,5,8,9,10,11,12,13,14,15,17,19,20,21,30,24,25,27,29,36],process_item:[2,9],dmozitem:12,simplejson:3,statscol:[5,38],accuraci:24,forbidden:19,log_fil:[37,5],which:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,30,24,25,26,27,28,29,31,32,34,35,36,37,38,39],webservice_port:18,singl:[10,29,0,2,19,13,20,5,39,14,15,25,26,17,27],uppercas:26,googlesitemap_en:17,unless:[29,0,1,20,5,15,26,17,9],integerfield:33,finsh:13,who:4,discov:[20,34,28,14,26],eucgb2312:5,downloaderstat:[5,19],deploi:[11,34,21,36],sitemap_rul:20,why:[30,5,38,21,15,17],marshal:7,latin1:5,url:[29,21,12,36,19,13,4,5,14,6,34,24,25,20,26,35,18,28],request:[10,11,21,1,12,29,13,20,5,30,19,15,6,35,25,36,26,22,18,28],uri:[20,7,24,19],face:4,inde:12,"_in":0,deni:[20,25],engine_start:30,webservice_logfil:18,fact:[21,12,28],dbm:19,text:[0,12,36,4,5,14,34,24,25,20,35],verbos:5,process_link:20,memusage_warning_mb:[5,17],retry_http_cod:19,textual:24,locat:[0,20,5,15,6,24,36],nois:29,should:[0,1,2,5,10,28,13,15,26,18,19,20,22,24,25,27,12,29,36,37,38,39],bytyp:28,suppos:[0,28,24],extract_regex:0,"7d97e98f8af710c7e7fe703abc8f639e0ee507c4":13,local:[12,13,14,7,35,36,27],export_empty_field:9,meant:[10,29,0,19,34,35,25,26],contribut:[15,11,36],familiar:[23,35,8,36,12],xmlrespons:26,htmlrespons:[26,28],autom:[0,36],acces:[5,26,0,8],smtppass:31,image_info_or_error:13,increas:[15,28],unstructur:8,organ:0,cmd_get_spider_stat:18,lala:8,gbk:5,imagespipelin:13,stuff:35,image_url:13,partit:28,contain:[0,1,2,4,5,7,8,9,11,12,13,14,15,26,17,18,19,20,23,24,25,16,32,34,35,36,38],urllengthmiddlewar:[29,5],scrapyd:[11,34,21,36],project_nam:34,nolink:34,knowledg:28,sgml:[4,25,20],smtpuser:31,troubl:11,engine_stop:[30,38],defaultheadersmiddlewar:[5,19],closer:[29,19],default_selector_class:0,pattern:[4,25],sitemapspid:20,thumb:13,spider_is_idl:27,state:[10,30,27],boutel:5,progress:[16,12],neither:28,entiti:24,thumbnail:13,image_path:13,kei:[10,29,0,38,19,13,20,5,32,7,8,26,17],tempfil:36,job:[10,11,26],entir:[21,19,12,39,9],monitor:[11,17,18,14,36],has_capac:27,equal:26,etc:[11,0,1,19,13,5,32,15,21,36,26,27],instanc:[28,20,33,26,17,9],dupefilt:5,libxml2docu:28,stats_spider_open:38,subclas:28,guidelin:15,arriv:[13,20],walk:12,ipython:[35,12],rpc:18,discount_expiration_d:8,respect:[5,19,9],restrict_xpath:25,quit:[26,7,21,17,28],addition:2,spiders_prod:5,compos:0,cb_kwarg:[20,28],json:[12,21,34,14,7,36,9,18,2],slashdot:35,pickleitemexport:[7,9],immedi:19,get_wsurl:18,cmd_list_resourc:18,dont_filt:[26,29,20],togeth:0,scheduler_disk_queu:21,furnitur:34,present:[13,15,33,26,9],replic:8,parse_nod:20,plain:[21,8,36,19],defin:[5,7,8,9,10,11,12,13,14,26,17,18,19,20,25,27,29,31,33,34,36,38],telnetconsole_en:[5,17],spider_middlewar:[29,5],duplicatespipelin:2,almost:38,libxml2:[24,17,3],site:[10,29,0,1,12,19,13,20,5,23,14,21,26],"1ca5879492b8fd606df1964ea3c1e2f4520f076f":13,product_nam:0,revis:[16,36],downloadtimeoutmiddlewar:[5,19],cleans:[6,2,14],parti:28,member:26,handl:[29,38,19,20,21,14,6,34,35,36,17],smarter:36,no_proxi:19,memusage_en:[5,17],failur:[13,26,30],inc:12,infer:[26,5],difficult:[21,28],http:[1,3,4,5,10,11,12,13,14,26,17,18,19,20,21,23,24,25,28,29,32,34,35,36],hostnam:38,xmlitemexport:[7,9],xpathselector:[0,12,24,20,35,28],logfil:37,php:[20,35,28,26],stats_dump:5,weher:18,off:[29,36,19],sre_pattern:28,colour:[4,0],feed_storages_bas:7,well:[5,28,20,21,14,15,24,36,9],thought:[15,34],exampl:[0,2,4,5,7,8,9,10,11,12,13,14,15,26,17,18,19,20,21,22,23,24,25,27,28,29,30,31,33,34,35,36,37,38,39],command:[10,11,21,12,3,5,32,34,35,36,37,17,18,28],productload:0,choos:[34,21],sibl:4,latest:[11,32,36,3],retry_en:19,paus:[10,11,27],pickled_meta:19,less:[20,21,5,28],templates_dir:5,obtain:[33,20,12],tcp:[27,36],simultan:5,web:[11,0,1,12,19,20,5,14,6,21,24,8,36,26,35,17,18,25],tight:18,priorit:29,hall:12,add:[29,0,1,2,3,13,4,19,38,32,30,33,34,35,11,8,36,26,17,27,12],book:12,spider_open:[30,28,17,2,9],match:[20,24,25],gmt:[34,19],rememb:[15,26,34,36],dest:18,robotstxt_en:5,know:[29,0,1,12,3,4,21,30,15,11,8,28,9],burden:0,press:[10,17],remove_ent:0,height:13,desc:12,insert:[29,17,19],resid:[34,36],like:[0,1,2,5,6,7,8,9,10,11,12,13,14,26,17,19,20,21,24,27,28,29,31,33,34,35,36],lost:[37,19],handle_httpstatus_list:[29,26],closespider_timeout:17,downloader_middlewares_bas:[5,19],active_s:27,lose:0,page:[29,0,1,12,3,13,4,25,5,34,14,6,19,24,11,8,20,26,35,17,21],default_request_head:[21,5,19],crawler:[11,5,28,21,31,14,18,27],diningc:19,exceed:22,enginestatusresourc:18,captur:[12,36],cmdarg:18,gayotspid:27,"export":[11,12,21,14,7,9,2],"0x3c44a10":35,b9628c4ab9b595f72f280b90c4fd093d:13,proper:[5,0,24],enviro:19,librari:[12,19,13,28,21,31,7,24,26,9],some_spid:38,est:27,leak:[11,21,17,28,27],avoid:[29,21,28,19,13,20,5,31,36,26,17],thank:14,overlap:16,leav:35,statsresourc:18,weslei:12,throttl:[21,5],importerror:[21,36],process_spider_output:29,usag:[29,0,28,19,13,38,5,22,34,26,17,18,27],host:[29,36,18,27,31],isn:[8,9],obei:19,although:[26,31],feed_format:7,filter_world:0,simpler:[4,8],about:[11,12,19,13,20,5,30,34,14,15,6,23,24,8,28,38,17,27,2],rare:39,socket:38,column:9,httpauthmiddlewar:[21,5,19],netrc:36,spiders_crawl:38,webservice_resourc:18,discard:29,disabl:[29,1,19,13,38,21,31,22,36,26,17,18,27],own:[29,0,2,36,19,20,5,31,14,33,24,8,26,16,17,18,25],process_spider_input:29,easy_instal:[28,3],automat:[0,38,3,13,20,5,14,21,35,36,26,17],track_ref:28,merg:[29,28,19,15,16,26],chunkedtransfermiddlewar:[5,19],transfer:[27,19],somearg:10,intention:14,httpcache_storag:19,arg1:36,"var":36,vat:2,parse_row:20,"function":[0,5,6,7,8,9,10,11,12,14,15,26,17,18,19,20,21,30,35,25,27,28,29,36,37,38,39],mailer:31,netherland:19,xmlfeedspid:20,brand:19,basic:[11,12,19,4,21,33,34,36,20,38,17,28],keyerror:[38,8],needs_backout:27,robotstxt_cachedir:5,startproject:[34,5,12],eas:[0,1,14,36],bug:[11,3,21,32,15,34],count:[38,28],mail_from:31,listspid:36,made:[11,5,18,19],whether:[29,19,5,7,25,36,38,9],httpcache_dbm_modul:19,displai:35,max_items_scrap:38,asynchron:6,below:[30,0,28,20,5,15,6,34,24,25,36,26,38,18,9],limit:[29,5,19],otherwis:[30,38,36,13,4,5,34,15,23,24,20,37,26,17,9],problem:[11,0,28,19,13,20,21,36,26],sitemap_url:20,succe:26,item_drop:30,"int":[26,5,31,18,9],dure:[5,14],pid:[17,28],twist:[30,3,13,21,31,15,6,34,36,37,26,18],implement:[34,21,12,19,13,20,5,38,31,15,6,7,24,25,36,37,26,9,17,18,2],rule:[11,0,4,14,34,25,20],userag:[5,19],pip:3,itemproc_s:27,probabl:[21,28,36],hpy:[28,27],jobsbot:35,percent:26,detail:[30,1,12,5,15,6,23,24,28],cmd_stop:18,other:[0,1,3,4,5,7,8,9,10,12,13,14,15,26,17,18,19,20,21,24,25,28,29,33,35,36,37,38],lookup:19,futur:[28,19,36,16,37,26,17],branch:[39,32],discount_perc:8,stat:[29,11,38,5,14,19,17,18,27],repeat:6,star:35,stop_on_non:0,debian:3,stai:[34,8,28],experienc:28,sphinx:15,parse_oth:20,download_delai:[21,5,34,36],get_xpath:0},objtypes:{"0":"std:command","1":"std:setting","2":"std:signal","3":"std:reqmeta","4":"py:module","5":"py:class","6":"py:method","7":"py:function","8":"py:exception","9":"py:attribute","10":"py:data","11":"py:classmethod"},titles:["Item Loaders","Using Firefox for scraping","Item Pipeline","Installation guide","Using Firebug for scraping","Settings","Architecture overview","Feed exports","Items","Item Exporters","Jobs: pausing and resuming crawls","Scrapy  documentation","Scrapy Tutorial","Downloading Item Images","Scrapy at a glance","Contributing to Scrapy","Experimental features","Extensions","Web Service","Downloader Middleware","Spiders","Frequently Asked Questions","Exceptions","Examples","XPath Selectors","Link Extractors","Requests and Responses","Telnet Console","Debugging memory leaks","Spider Middleware","Signals","Sending e-mail","Ubuntu packages","DjangoItem","Command line tool","Scrapy shell","Scrapy Service (scrapyd)","Logging","Stats Collection","Versioning and API Stability"],objnames:{"0":["std","command","command"],"1":["std","setting","setting"],"2":["std","signal","signal"],"3":["std","reqmeta","reqmeta"],"4":["py","module","Python module"],"5":["py","class","Python class"],"6":["py","method","Python method"],"7":["py","function","Python function"],"8":["py","exception","Python exception"],"9":["py","attribute","Python attribute"],"10":["py","data","Python data"],"11":["py","classmethod","Python class method"]},filenames:["topics/loaders","topics/firefox","topics/item-pipeline","intro/install","topics/firebug","topics/settings","topics/architecture","topics/feed-exports","topics/items","topics/exporters","topics/jobs","index","intro/tutorial","topics/images","intro/overview","contributing","experimental/index","topics/extensions","topics/webservice","topics/downloader-middleware","topics/spiders","faq","topics/exceptions","intro/examples","topics/selectors","topics/link-extractors","topics/request-response","topics/telnetconsole","topics/leaks","topics/spider-middleware","topics/signals","topics/email","topics/ubuntu","experimental/djangoitems","topics/commands","topics/shell","topics/scrapyd","topics/logging","topics/stats","versioning"]})