net.py 文件源码

def urlopen(url, headers={}, data=None, retries=RETRIES, timeout=TIMEOUT):
    '''????http??, ???Request.

    headers ???dict. ?????????, ??User-Agent, Referer?, ?
    ????????.

    ????????http??, ??????????.
    ???????gzip????, ????gzip???????, ???????
    ??.
    req.data ?????????http????, ????UTF-8?????.
    '''
    headers_merged = default_headers.copy()
    for key in headers.keys():
        headers_merged[key] = headers[key]
    opener = urllib.request.build_opener(ForbiddenHandler)
    opener.addheaders = [(k, v) for k,v in headers_merged.items()]

    for i in range(retries):
        try:
            req = opener.open(url, data=data, timeout=timeout)
            encoding = req.headers.get('Content-encoding')
            req.data = req.read()
            if encoding == 'gzip':
                req.data = gzip.decompress(req.data)
            elif encoding == 'deflate':
                req.data = zlib.decompress(req.data, -zlib.MAX_WBITS)
            return req
        except OSError:
            logger.error(traceback.format_exc())

        except:
            logger.error(traceback.format_exc())

    return None