python类PROXIES的实例源码-面圈网

middlewares.py 文件源码项目：spider_scrapy_lianjia 作者: stamhe 项目源码文件源码阅读 24 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        request.meta['proxy'] = "http://http-dyn.abuyun.com:9020"
        # ??????????????????????????
        proxy_user_pass = "HX4OS8905X90183D:C3127131D399C059"
        encoded_user_pass = base64.encodestring(proxy_user_pass)
        request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        #print "proxy = %s" (request.meta['proxy'])

        return

        proxy = random.choice(PROXIES)
        request.meta['proxy'] = "http://%s" % proxy
        return

        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy['ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

        return

rotate_useragent.py 文件源码项目：NewsScrapy 作者: yinzishao 项目源码文件源码阅读 21 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy['ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

middlewares.py 文件源码项目：finance_news_analysis 作者: pskun 项目源码文件源码阅读 21 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers[
                'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            # print "**************ProxyMiddleware have pass************" +
            # proxy['ip_port']
        else:
            # print "**************ProxyMiddleware no pass************" +
            # proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
        pass

middlewares.py 文件源码项目：taobao 作者: laogewen 项目源码文件源码阅读 29 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'].encode(encoding='utf-8'))
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass.decode()
            print ("**************ProxyMiddleware have pass************" + proxy['ip_port'])
        else:
            print ("**************ProxyMiddleware no pass************" + proxy['ip_port'])
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

middleware.py 文件源码项目：twitter_scrapy 作者: muscledreamer 项目源码文件源码阅读 27 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            request.meta['proxy'] = "https://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy['ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            print "?????"

middlewares.py 文件源码项目：ScrapyImage 作者: donnki 项目源码文件源码阅读 24 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy['ip_port']
        else:
            print "**************ProxyMiddleware no pass************http://%s" % proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

middlewares.py 文件源码项目：YelpCrawlSpider 作者: yjp999 项目源码文件源码阅读 24 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        else:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

proxy_middleware.py 文件源码项目：scrapy-cluster 作者: WalnutATiie 项目源码文件源码阅读 23 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        request.meta['proxy'] = "http://%s" % proxy['ip_port']
        #print '************using proxy:'+proxy['ip_port']

middlewares.py 文件源码项目：tutorial 作者: zhaobin022 项目源码文件源码阅读 30 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
    proxy = random.choice(PROXIES)
    if proxy['user_pass'] is not None:
      request.meta['proxy'] = "http://%s" % proxy['ip_port']
      encoded_user_pass = base64.encodestring(proxy['user_pass'])
      request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
      print "**************ProxyMiddleware have pass************" + proxy['ip_port']
    else:
      print "**************ProxyMiddleware no pass************" + proxy['ip_port']
      request.meta['proxy'] = "http://%s" % proxy['ip_port']

middlewares.py 文件源码项目：scrapy_sight 作者: wankaiss 项目源码文件源码阅读 26 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            # print "**************ProxyMiddleware have pass************" + proxy['ip_port']
        else:
            # print "**************ProxyMiddleware no pass************" + proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

middlewares.py 文件源码项目：badoo_scrapy_splash_redis 作者: Supe2015 项目源码文件源码阅读 28 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass            
        print "**************ProxyMiddleware have pass************" + proxy['ip_port']
        else:
        print "**************ProxyMiddleware no pass************" + proxy['ip_port']
            request.meta['proxy'] = "http://%s" % proxy['ip_port']

middlewares.py 文件源码项目：DianpingSpider 作者: tensorchen 项目源码文件源码阅读 27 收藏 0 点赞 0 评论 0

def process_request(self, request, spider):
        request.meta['proxy'] = "http://%s" % random.choice(PROXIES)
#        proxy_user_pass = "tensor:92598866"

#        encode_user_pass = base64.encodestring(proxy_user_pass)
#        request.headers['Proxy-Authorization'] = 'Basic' + proxy_user_pass