middlewares.py 文件源码

def process_request(self, request, spider):
        if not request.url:
            return None
        url_hash = hashlib.md5(request.url.encode("utf8")).hexdigest()
        if self.redis_client.sismember(spider.name, url_hash):
            raise IgnoreRequest("Spider : %s, IgnoreRequest : %s" % (spider.name, request.url))
        else:
            self.redis_client.sadd(spider.name, url_hash)