s3_util.py 文件源码

def _native_download_file(meta, full_dst_file_name, max_concurrency):
        logger = getLogger(__name__)
        try:
            akey = SnowflakeS3Util._get_s3_object(meta, meta[u'src_file_name'])
            akey.download_file(
                full_dst_file_name,
                Callback=meta[u'get_callback'](
                    meta[u'src_file_name'],
                    meta[u'src_file_size'],
                    output_stream=meta[u'get_callback_output_stream']) if
                        meta[u'get_callback'] else None,
                Config=TransferConfig(
                    multipart_threshold=SnowflakeS3Util.DATA_SIZE_THRESHOLD,
                    max_concurrency=max_concurrency,
                    num_download_attempts=10,
                )
            )
            meta[u'result_status'] = ResultStatus.DOWNLOADED
        except botocore.exceptions.ClientError as err:
            if err.response[u'Error'][u'Code'] == EXPIRED_TOKEN:
                meta[u'result_status'] = ResultStatus.RENEW_TOKEN
            else:
                logger.debug(
                    u"Failed to download a file: %s, err: %s",
                    full_dst_file_name, err, exc_info=True)
                raise err
        except RetriesExceededError as err:
            meta[u'result_status'] = ResultStatus.NEED_RETRY
            meta[u'last_error'] = err
        except OpenSSL.SSL.SysCallError as err:
            if err.args[0] not in (
                    ERRORNO_WSAECONNABORTED,
                    errno.ECONNRESET,
                    errno.ETIMEDOUT,
                    errno.EPIPE,
                    -1):
                raise err
            meta[u'last_error'] = err
            if err.args[0] == ERRORNO_WSAECONNABORTED:
                # connection was disconnected by S3
                # because of too many connections. retry with
                # less concurrency to mitigate it

                meta[u'result_status'] = ResultStatus.NEED_RETRY_WITH_LOWER_CONCURRENCY
            else:
                meta[u'result_status'] = ResultStatus.NEED_RETRY