lambda_function.py 文件源码

def lambda_handler(event, context):
    '''Invoke Lambda '''
    # load config from json file in s3 bucket
    config = load_config(context)

    # create ES connection with sts auth file
    es_client = Elasticsearch(host=config['es_host'],
                              port=80,
                              connection_class=RequestsHttpConnection,
                              http_auth=sts_auth(config),
                              timeout=config['es_connection_timeout'])

    # create new index with custom mappings from config, ignore if it's already created
    # new index will be created for everyday YMV
    suffix = datetime.strftime(datetime.now(), '%Y-%m-%d')
    resp = es_client.indices.create(index="cloudfrontlog-" +
                                    suffix, body=config['es_mapping'],
                                    ignore=400)
    print resp

    # create a s3 boto client
    s3_client = boto3.client('s3')

    # split bucket and filepath to variables
    bucket = event['Records'][0]['s3']['bucket']['name']
    key = event['Records'][0]['s3']['object']['key']

    # set the file path
    file_path = '/tmp/cflogfile.gz'

    # download the gzip log from s3
    s3_client.download_file(bucket, key, file_path)

    # parse the log
    record_set = parse_log('/tmp/cflogfile.gz')

    # write the dict to ES
    resp = write_bulk(record_set, es_client, config)
    print resp