retriever.py 文件源码

def get_pdf(pdf_link):

    # check whether value already existing in permanent storage:
    pdf_name = pdf_link.rsplit('/', 1)[-1]  # set filename according to last element of link
    if not check_db(pdf_name) and not check_db(pdf_link):
        # print 'Downloading: {}'.format(pdf_link)
        try:
            opener = urllib2.build_opener()
            opener.addheaders = [('User-agent', USER_AGENT)]

            r = opener.open(pdf_link)

            path = tmp_dir + pdf_name

            with open(path, "wb") as code:  # 'w'
                code.write(r.read())

            # log successful download:
            log_download('DOWNLOADED: {}'.format(pdf_link))

        except Exception as e:
            log_download('FAILURE: {} | {}'.format(pdf_link, e))
    else:
        log_download('File already downloaded: {}'.format(pdf_name))