def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from boto.gs.connection import GSConnection
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is
# enabled).
#
# The same is not true for S3's HTTPS certificates. In fact,
# we don't want to do this for S3 because S3 requires the
# subdomain to match the location of the bucket. If the proper
# subdomain is not used, the server will return a 301 redirect
# with no Location header.
#
# Note: the following import can't be moved up to the
# start of this file else it causes a config import failure when
# run from the resumable upload/download tests.
from boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
python类storage_uri()的实例源码
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from boto.gs.connection import GSConnection
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is
# enabled).
#
# The same is not true for S3's HTTPS certificates. In fact,
# we don't want to do this for S3 because S3 requires the
# subdomain to match the location of the bucket. If the proper
# subdomain is not used, the server will return a 301 redirect
# with no Location header.
#
# Note: the following import can't be moved up to the
# start of this file else it causes a config import failure when
# run from the resumable upload/download tests.
from boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is enabled).
# Note: the following import can't be moved up to the start of
# this file else it causes a config import failure when run from
# the resumable upload/download tests.
from boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from boto.gs.connection import GSConnection
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
def multipart_upload_worker(scheme, parsed_url, storage_uri, bucket_name, multipart_id,
filename, offset, bytes, num_retries, queue):
"""
Worker method for uploading a file chunk to S3 using multipart upload.
Note that the file chunk is read into memory, so it's important to keep
this number reasonably small.
"""
def _upload_callback(uploaded, total):
worker_name = multiprocessing.current_process().name
log.Debug("%s: Uploaded %s/%s bytes" % (worker_name, uploaded, total))
if queue is not None:
queue.put([uploaded, total]) # Push data to the consumer thread
def _upload(num_retries):
worker_name = multiprocessing.current_process().name
log.Debug("%s: Uploading chunk %d" % (worker_name, offset + 1))
try:
conn = get_connection(scheme, parsed_url, storage_uri)
bucket = conn.lookup(bucket_name)
for mp in bucket.list_multipart_uploads():
if mp.id == multipart_id:
with FileChunkIO(filename, 'r', offset=offset * bytes, bytes=bytes) as fd:
start = time.time()
mp.upload_part_from_file(fd, offset + 1, cb=_upload_callback,
num_cb=max(2, 8 * bytes / (1024 * 1024))
) # Max num of callbacks = 8 times x megabyte
end = time.time()
log.Debug(("{name}: Uploaded chunk {chunk}"
"at roughly {speed} bytes/second").format(name=worker_name,
chunk=offset + 1,
speed=(bytes / max(1, abs(end - start)))))
break
conn.close()
conn = None
bucket = None
del conn
except Exception as e:
traceback.print_exc()
if num_retries:
log.Debug("%s: Upload of chunk %d failed. Retrying %d more times..." % (
worker_name, offset + 1, num_retries - 1))
return _upload(num_retries - 1)
log.Debug("%s: Upload of chunk %d failed. Aborting..." % (
worker_name, offset + 1))
raise e
log.Debug("%s: Upload of chunk %d complete" % (worker_name, offset + 1))
return _upload(num_retries)
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is enabled).
# Note: the following import can't be moved up to the start of
# this file else it causes a config import failure when run from
# the resumable upload/download tests.
from boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from boto.gs.connection import GSConnection
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from boto.gs.connection import GSConnection
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is
# enabled).
#
# The same is not true for S3's HTTPS certificates. In fact,
# we don't want to do this for S3 because S3 requires the
# subdomain to match the location of the bucket. If the proper
# subdomain is not used, the server will return a 301 redirect
# with no Location header.
#
# Note: the following import can't be moved up to the
# start of this file else it causes a config import failure when
# run from the resumable upload/download tests.
from boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is enabled).
# Note: the following import can't be moved up to the start of
# this file else it causes a config import failure when run from
# the resumable upload/download tests.
from boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from boto.gs.connection import GSConnection
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
