maintainers.py 文件源码

def _test_proxy_alive(self, host, port, protocol, proxy_type,
                          url=b'http://www.baidu.com', timeout=10):
        endpoint = TCP4ClientEndpoint(reactor, host, int(port))
        agent = ProxyAgent(endpoint)
        d = agent.request(b'GET', url)
        self.currents += 1
        proxy = '{}:{}'.format(host, port)
        key = 'proxy_info:' + proxy

        if proxy_type == 'rookies_checking':
            def _callback(ignored):
                pipe = self.conn.pipeline(False)
                pipe.zrem('rookies_checking', proxy)
                pipe.hset(key, 'failed_times', 0)
                # Move proxy from rookies to availables
                pipe.smove('rookie_proxies', 'available_proxies',
                           '{}://{}'.format(protocol, proxy))
                pipe.zadd('availables_checking', proxy, time.time() + 30)
                pipe.execute()

            def _errback(err):
                if self.conn.hincrby(key, 'failed_times', 1) < 3:
                    # If not reach the maximum of failed_times
                    # Since it is not important so re-check it after 10 seconds
                    self.conn.zadd('rookies_checking', proxy, time.time() + 10)
                else:
                    pipe = self.conn.pipeline(False)
                    pipe.zrem('rookies_checking', proxy)
                    pipe.smove('rookie_proxies', 'dead_proxies',
                               '{}://{}'.format(protocol, proxy))
                    pipe.execute()
        else:
            def _callback(ignored):
                pipe = self.conn.pipeline(False)
                pipe.hset(key, 'failed_times', 0)
                pipe.zadd('availables_checking', proxy, time.time() + 30)
                pipe.smove('lost_proxies', 'available_proxies',
                           '{}://{}'.format(protocol, proxy))
                pipe.execute()

            def _errback(err):
                pipe = self.conn.pipeline(False)
                if self.conn.hincrby(key, 'failed_times', 1) < 3:
                    pipe.zadd('availables_checking', proxy, time.time() + 10)
                    pipe.smove('available_proxies', 'lost_proxies',
                               '{}://{}'.format(protocol, proxy))
                else:
                    pipe.zrem('availables_checking', proxy)
                    pipe.smove('lost_proxies', 'dead_proxies',
                               '{}://{}'.format(protocol, proxy))
                    pipe.delete(key)
                pipe.execute()

        d.addCallbacks(_callback, _errback)
        reactor.callLater(timeout, d.cancel)

        def _clean(ignored):
            self.currents -= 1

        d.addBoth(_clean)