aiqiyi_spider.py 文件源码

def __init__(self):
        scrapy.spiders.Spider.__init__(self)

        self.global_settings = get_project_settings()
        if self.global_settings['PLATFORM'] in ['win', 'mac']:
            self.driver = webdriver.PhantomJS(executable_path= self.global_settings['PHANTOMJS_PATH'])
        elif self.global_settings['PLATFORM'] in ['linux']:
            self.driver = webdriver.PhantomJS()
        self.driver.set_page_load_timeout(30)
        self.driver.implicitly_wait(10)

        self.type_id_list = self.global_settings['CRAWLER']['type_id_list']
        self.re_type_id = re.compile(self.global_settings['CRAWLER']['re_type_id'])
        self.url_template = self.global_settings['CRAWLER']['url_template']