scraper.py 文件源码

def update_single_feed(worker, job):
    log(0, "'update-single-feed' initiated")

    try:
        request = bson.BSON(job.data).decode()
        url = request['url']
    except:
        log(2, "Invalid parameters provided")
        return str(bson.BSON.encode({
            'status': 'error',
            'error-description': 'Invalid parameters',
            }))

    if key is not None:
        if 'key' not in request or request['key'] != key:
            log(2, "Secret key mismatch")
            response = bson.BSON.encode({
                'status': 'error',
                'description': 'Secret key mismatch',
                })
            return str(response)

    try:
        feed = get_single_feed_doc(url)
        updated_feeds = gather_updates(feed[0])
        update_database(feed[0], updated_feeds)
    except Exception as e:
        log(2, "'update-single-feed' failed")
        log(2, str(e))
        return str(bson.BSON.encode({
            "status": "error",
            "error-description": str(e)
        }))

    log(0, "'update-single-feed' finished")
    return str(bson.BSON.encode({
        "status": "ok",
        "updated_feeds": [x['_id'] for x in feed],
    }))


# updates all of the item fields for all the unique feeds in the feeds db