def cli(ctx, **kwargs):
"""A crawler system."""
logging.config.fileConfig(kwargs['logging_config'])
config = {}
config_filepath = kwargs['config']
if config_filepath:
if not os.path.exists(config_filepath):
raise IOError('No such file or directory: "%s".' % config_filepath)
if not os.path.isfile(config_filepath):
raise IOError('Is not a file: "%s".' % config_filepath)
try:
with open(config_filepath, 'r') as f:
config = yaml.load(f)
except Exception as err:
raise err
if kwargs.get('redis'):
redis_conn = utils.connect_redis(kwargs['redis'])
elif config.get('redis'):
redis_conn = utils.connect_redis(config['redis']['url'])
else:
raise Exception('Could not find redis address.')
mongodb_conn = None
if kwargs.get('mongodb'):
mongodb_conn = utils.connect_mongodb(kwargs['mongodb'])
elif config.get('mongodb'):
mongodb_conn = utils.connect_mongodb(config['mongodb']['url'])
else:
logging.warning('Could not find mongodb address. No results will be saved.')
from fulmar.utils import LUA_RATE_LIMIT_SCRIPT
lua_rate_limit = redis_conn.register_script(LUA_RATE_LIMIT_SCRIPT)
setattr(utils, 'redis_conn', redis_conn)
setattr(utils, 'lua_rate_limit',lua_rate_limit)
setattr(utils, 'mongodb_conn', mongodb_conn)
ctx.obj = utils.ObjectDict(ctx.obj or {})
ctx.obj.update(config)
return ctx
评论列表
文章目录