collect_data.py 文件源码

def dynamically_scrape_data(filename, link_list, interval, num_retries = 10):
    """
    Repeatedly runs the scraper every time the specified interval has passed
    and continuously appends the data to a file.
    """

    def job():
        scrape_all_data_from_all_featured_products(filename,
                                                   link_list,
                                                   num_retries)
    job()
    schedule.every(interval).minutes.do(job)

    while True:
        schedule.run_pending()
        time.sleep(30)

    print "Dynamic scraping finished"