cli.py 文件源码

def cli(ctx, **kwargs):
    """A crawler system."""
    logging.config.fileConfig(kwargs['logging_config'])
    config = {}
    config_filepath = kwargs['config']
    if config_filepath:
        if not os.path.exists(config_filepath):
            raise IOError('No such file or directory: "%s".' % config_filepath)
        if not os.path.isfile(config_filepath):
            raise IOError('Is not a file: "%s".' % config_filepath)
        try:
            with open(config_filepath, 'r') as f:
                config = yaml.load(f)
        except Exception as err:
            raise err

    if kwargs.get('redis'):
        redis_conn = utils.connect_redis(kwargs['redis'])
    elif config.get('redis'):
        redis_conn = utils.connect_redis(config['redis']['url'])
    else:
        raise Exception('Could not find redis address.')

    mongodb_conn = None
    if kwargs.get('mongodb'):
        mongodb_conn = utils.connect_mongodb(kwargs['mongodb'])
    elif config.get('mongodb'):
        mongodb_conn = utils.connect_mongodb(config['mongodb']['url'])
    else:
        logging.warning('Could not find mongodb address. No results will be saved.')

    from fulmar.utils import LUA_RATE_LIMIT_SCRIPT
    lua_rate_limit = redis_conn.register_script(LUA_RATE_LIMIT_SCRIPT)

    setattr(utils, 'redis_conn', redis_conn)
    setattr(utils, 'lua_rate_limit',lua_rate_limit)
    setattr(utils, 'mongodb_conn', mongodb_conn)

    ctx.obj = utils.ObjectDict(ctx.obj or {})
    ctx.obj.update(config)
    return ctx