Commit 890f5ac40e64fd9b28a5fc3ede0509fee7abbe57
Ticket #2:Disbale http_caching. Index page was also being cached, which means
that whenever the spider ran, it got only a cached version of the page,
hence will not get new posts.
Setting caching policy to be compliant with RFC2616 does not help, the
pages being served from the web server do not have any cache-control
directives.
Fix: Using anydbm module to implement an equivalent of caching, maintain a db
of urls which have been crawled and posted, do not process those urls
again.
TODO: Find a more idiomatic way of doing this. This can move to sweets, maybe.
| | |
| 5 | 5 | include/ |
| 6 | 6 | local/ |
| 7 | 7 | lib/ |
| build/ |
| 8 | 9 | *.db |
| 9 | 10 | *.pid |
| 10 | 11 | conf.py |
| urlCache |
| | |
| 13 | 13 | |
| 14 | 14 | # Crawl responsibly by identifying yourself (and your website) on the user-agent |
| 15 | 15 | #USER_AGENT = 'postScraper (+http://www.yourdomain.com)' |
| HTTPCACHE_ENABLED = True |
| # HTTPCACHE_ENABLED = True |
| #HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.RFC2616Policy' |
| # SPIDER_MIDDLEWARES = { |
| # 'postScraper.middlewares.deltafetch.DeltaFetch': 100, |
| # } |
|
| # DELTAFETCH_ENABLED = True |
| # DOTSCRAPY_ENABLED = True |
| | |
| 3 | 3 | from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor |
| 4 | 4 | from scrapy.selector import Selector |
| 5 | 5 | from scrapy.contrib.loader import ItemLoader |
|
| 7 | 6 | from postScraper.items import PostscraperItem |
| 8 | 7 | |
| 9 | 8 | import facebook |
| 10 | 9 | import conf |
| import anydbm |
| 11 | 11 | |
| 12 | 12 | |
| 13 | 13 | class SwaraSpider(CrawlSpider): |
| … | … | |
| 18 | 18 | callback='parse_start'),) |
| 19 | 19 | |
| 20 | 20 | def parse_start(self, response): |
| if 'cached' not in response.flags: |
| db = anydbm.open('urlCache', 'c') |
| if response.url not in db.keys(): |
| 22 | 23 | xpath = Selector() |
| 23 | 24 | loader = ItemLoader(item=PostscraperItem(), response=response) |
| 24 | 25 | |
| … | … | |
| 38 | 38 | description=content[0]['content'].encode('utf8'), |
| 39 | 39 | message="#CGNetSwara http://cgnetswara.org/" + |
| 40 | 40 | content[1]['audio']) |
| print str(response.url) |
| print type(response.url) |
| db[response.url] = str(True) |
| db.close() |
| else: |
| print "Not posting content from " + response.url |
| db.close() |
