proc.py 文件源码

def start_requests(self):
        # while len(self.finished) < len(self.all_urls):
        current_hour = time.strftime("%Y%m%d%H", time.localtime())
        if current_hour != START_HOUR:
            self.logger.info("It's already %s. Stopping..." % current_hour)
            return
        for url, item_idx in self.all_urls.iteritems():
            if not self.cookies:
                raise CloseSpider("No enough cookies.")
            if item_idx in self.finished:
                continue
            else:
                yield Request(url, callback=self.parse_item)
                # self.logger.info(u'Crawled %s / %s. Done :)' % (len(self.finished), len(self.all_urls)))