worker.py 文件源码

def main():
    """Creates a redis client, a YARN ResourceManager REST API client, and a YARN
    poller that puts information about the YARN cluster and its applications into
    redis on a timed interval.
    """
    log_level = os.getenv('LOG_LEVEL', 'INFO')
    logging.basicConfig(level=getattr(logging, log_level))

    host, port = os.environ['REDIS_ENDPOINT'].split(":")
    redis_client = redis.StrictRedis(host=host, port=port)
    yarn_handler = YARNHandler(os.environ['YARN_ENDPOINT'])

    ym = YARNPoller(redis_client, yarn_handler)
    ym.register_handler("SPARK", SparkHandler)
    ym.register_handler("MAPREDUCE", MapredHandler)
    ym.register_handler("MAPRED", MapredHandler)
    ym.loop(int(os.environ["YARN_POLL_SLEEP"]))