smSpider.py 文件源码

def parse_content(self,url):

        try:
            self.driver.get(url)
        except Exception,e:
            print "give up one detail"
            return ""

        try:
            element = WebDriverWait(self.driver, 30).until(
                EC.presence_of_all_elements_located((By.TAG_NAME, 'table'))
            )
            print 'element:\n', element
        except Exception, e:
            print Exception, ":", e
            print "wait failed"

        page_source = self.driver.page_source
        bs_obj = BeautifulSoup(page_source, "lxml")

        return '%s'%bs_obj.find('td', class_='a-content').p.get_text().encode('utf-8','ignore')