SignAndSearch.py 文件源码

def loadSearch(self, url, firstName='results'):
        """
        Loads the search page using the url provided and returns raw search results
        """
        print " inside loadSearch .."

        '''
        97.77.104.22:80
        174.129.204.124:80
        '''
        proxy = {
            "http":"209.222.25.83:3128",
        }
        headers = {'Accept-Encoding': 'identity'}
        html2 = requests.get(url, proxies=proxy, headers=headers)
        print "HTML 2"
        # print html2.content
        # html = html2.content
        html = self.loadPage(url)
        print "SPAGE"
        # print sPage[:200]
        spContent = BeautifulSoup(html)

        #title = spContent.find('title')
        #if title is not None:
            #if title.string is not lSrchTitle:
                #sys.exit('There is some problem with url provided, it does not correspond to Linkedin Search')
        comment = None
        comments = spContent.findAll(text=lambda text:isinstance(text, Comment))
        print "COMMENTS"
        # print comments
        # print " >> BEAUTIFULSOUP FINDALL"
        #print comments
        cLen = len(comments)
        print "Length of COmments"+cLen.__str__()
        if cLen > 0 and cLen > 11:
            comment = comments[11]  
        if comment is None:
            for cmnt in comments:   
                if firstName in cmnt:
                    comment = cmnt
        print "output COMMENTS :"
        # print comment            
        return comment