proxydb.py 文件源码

def parse_page(self, response):
        super(ProxyDBSpider, self).parse_page(response)

        data = response.xpath('//tbody/tr').extract()
        for i, d in enumerate(data):
            sel = Selector(text = d)

            ip_port = sel.xpath('//td/a/text()').extract_first()
            ip = ip_port.split(':')[0]
            port = ip_port.split(':')[1]
            country = sel.xpath('//td/img/@title').extract_first()
            anonymity = sel.xpath('//td/span[@class="text-success"]/text()').extract_first()

            proxy = Proxy()
            proxy.set_value(
                    ip = ip,
                    port = port,
                    country = country,
                    anonymity = anonymity,
                    source = self.name
            )

            self.add_proxy(proxy = proxy)