collect_data.py 文件源码

def clean_links_and_dynamically_scrape(filename, interval, num_retries = 10):
    """
    Repeatedly updates the link list and runs the scraper every time the
    specified interval has passed and continuously appends the data to a file.
    """

    def job():
        clean_links_and_scrape(filename, num_retries)

    job()
    schedule.every(interval).minutes.do(job)

    while True:
        schedule.run_pending()
        time.sleep(30)

    print "Dynamic scraping finished"