crawler.py 文件源码

def _get_response(self,url,method='get',headers={},files=[],data=None,cookies=None,cert=None,timeout=30,**kwargs):
        method=method.upper()
        #self._s.headers.update(headers)
        pre=requests.Request(method=method,url=url,data=data,files=files)
        prepped=self._s.prepare_request(pre)
        try:
            with self._s.send(prepped,stream=True,cert=cert,timeout=timeout) as resp:
                #self._header.parse_header(dict(resp.headers))
                #self._s.headers.update(self._header.get_default_header())
                #content_type=resp.headers.get('content-type')
                #encoding=self._get_content_encoding(content_type)
                #regx=re.compile('.*(text\/html|text\/xml).*')
                if resp.status_code==requests.codes.OK:
                    #Don't handle redirect url for now
                    '''
                    with open('temp.txt','wb') as f:
                        f.write(resp.content)
                    '''
                    self._handler.handle_data(resp)
                elif resp.status_code!=requests.codes.OK:
                    print("Connected url %s \t %d" % (url,resp.status_code))
                else:
                    #If the response is not html or xml ,save the url to redis 
                    pass
        except requests.ConnectTimeout,requests.ReadTimeout:
            print('Connect %s timeout .\n' % (url))
            if self._redis_enable:
                self._r.lpush(self._start,url)
            else:
                task.url_task.append(url)