NECIPSSpider_wo_scrapy.py 文件源码

def get_driver_phantomjs():
    """
    References:
    PhantomJS:
    1. [??PHANTOMJS?USER-AGENT](http://smilejay.com/2013/12/set-user-agent-for-phantomjs/)
    2. [Selenium 2 - Setting user agent for IE and Chrome](http://stackoverflow.com/questions/6940477/selenium-2-setting-user-agent-for-ie-and-chrome)
    """
    dcap = dict(DesiredCapabilities.PHANTOMJS)

    # Setting User-Agent
    ua = random.choice(RotateUserAgentMiddleware.user_agent_list)
    if ua:
        print("Current User-Agent is:", ua)
        dcap["phantomjs.page.settings.userAgent"] = ua

    driver = webdriver.PhantomJS(executable_path=r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs", desired_capabilities=dcap)

    """
    # Setting IP Proxies
    # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    ip_proxy = get_proxy()
    if ip_proxy:
        proxy.http_proxy = ip_proxy

    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    # proxy.add_to_capabilities(DesiredCapabilities.PHANTOMJS)
    # driver.start_session(DesiredCapabilities.PHANTOMJS)
    proxy.add_to_capabilities(dcap)
    driver.start_session(dcap)
    """

    # ??????
    driver.set_page_load_timeout(TIMEOUT)
    driver.set_script_timeout(TIMEOUT)  # ???????????

    return driver