crawl_page.py 文件源码

def safe_post_message(slacker, crawler, start_id, post_message, max_retries=3):
    """
    post for my slack channel
    :param slacker.Slacker slacker:
    :param g_crawler.crawler.GCrawler crawler:
    :param int start_id:
    :param str post_message:
    :param int max_retries=3:
    """

    retries = 0

    while True:
        try:
            slacker.chat.post_message("#crawler", "[{}] [ID: {}] {}".format(crawler.__class__.__name__, start_id, post_message))
        except HTTPError as http_err:
            retries += 1
            if retries >= max_retries:
                raise Exception("Too many retries.")

            wait = 2 ** (retries)
            print("[ RETRY ] Waiting {} seconds...".format(wait))
            time.sleep(wait)
        else:
            break