s3grouper.py 文件源码

def dl_and_parse(s3,headers,keylist,bucket):
    for key in keylist:
        with TemporaryFile() as fp:
            try:
                s3.Bucket(bucket).download_fileobj(key,fp)
            except ClientError as e:
                logger.error('Unable to download s3://{}/{}'.format(bucket, key))
                logger.debug('Received error: {}'.format(e))
                sys.exit(5)
            fp.seek(0)
            with TextIOWrapper(GzipFile(fileobj=fp,mode='r')) as f:
                try:
                    reader = csv.DictReader(f,fieldnames=headers,delimiter=',',quoting=csv.QUOTE_MINIMAL)
                    for row in reader:
                        yield row
                except csv.Error as e:
                    logger.error("Unable to read CSV '{}'".format(reader.line))
                    logger.debug('Received error: {}'.format(e))
                    sys.exit(3)

# main parser