CrawlBase.py 文件源码

def getListProxies(self):  
        session = requests.session()
        self.getRandomUserAgent()  
        page = session.get("http://www.xicidaili.com/nn", headers=self.headers)  
        soup = BeautifulSoup(page.text, 'lxml')  

        proxyList = []  
        taglist = soup.find_all('tr', attrs={'class': re.compile("(odd)|()")})  
        for trtag in taglist:  
            tdlist = trtag.find_all('td')  
            proxy = { 'https': tdlist[1].string + ':' + tdlist[2].string }  
            proxyList.append(proxy)  

        return proxyList

    #????????????????????