phantomspider.py 文件源码

def phantomjs_process(self,request):
        def do_counts(str_counts):
            try:
                counts = str_counts.replace(',','')
                return counts
            except:
                return 0
        def do_item(item):
            if item and isinstance(item,list):
                return item[0]
            return item
        try:
            url = request.url
            driver = webdriver.PhantomJS(executable_path="/usr/bin/phantomjs")
            driver.get(request.url)
            body = driver.page_source
            response = HtmlResponse(url,body=body.encode('UTF-8'),request=request)
        except Exception as e:
            self.logger.error("phantomjs error:",e,url)
            return []
        return self.parse_one_news(response)