Mini Shell
# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import math
from botocore.compat import six
from s3transfer.compat import seekable, readable
from s3transfer.futures import IN_MEMORY_UPLOAD_TAG
from s3transfer.tasks import Task
from s3transfer.tasks import SubmissionTask
from s3transfer.tasks import CreateMultipartUploadTask
from s3transfer.tasks import CompleteMultipartUploadTask
from s3transfer.utils import get_callbacks
from s3transfer.utils import get_filtered_dict
from s3transfer.utils import DeferredOpenFile, ChunksizeAdjuster
class AggregatedProgressCallback(object):
def __init__(self, callbacks, threshold=1024 * 256):
"""Aggregates progress updates for every provided progress callback
:type callbacks: A list of functions that accepts bytes_transferred
as a single argument
:param callbacks: The callbacks to invoke when threshold is reached
:type threshold: int
:param threshold: The progress threshold in which to take the
aggregated progress and invoke the progress callback with that
aggregated progress total
"""
self._callbacks = callbacks
self._threshold = threshold
self._bytes_seen = 0
def __call__(self, bytes_transferred):
self._bytes_seen += bytes_transferred
if self._bytes_seen >= self._threshold:
self._trigger_callbacks()
def flush(self):
"""Flushes out any progress that has not been sent to its callbacks"""
if self._bytes_seen > 0:
self._trigger_callbacks()
def _trigger_callbacks(self):
for callback in self._callbacks:
callback(bytes_transferred=self._bytes_seen)
self._bytes_seen = 0
class InterruptReader(object):
"""Wrapper that can interrupt reading using an error
It uses a transfer coordinator to propagate an error if it notices
that a read is being made while the file is being read from.
:type fileobj: file-like obj
:param fileobj: The file-like object to read from
:type transfer_coordinator: s3transfer.futures.TransferCoordinator
:param transfer_coordinator: The transfer coordinator to use if the
reader needs to be interrupted.
"""
def __init__(self, fileobj, transfer_coordinator):
self._fileobj = fileobj
self._transfer_coordinator = transfer_coordinator
def read(self, amount=None):
# If there is an exception, then raise the exception.
# We raise an error instead of returning no bytes because for
# requests where the content length and md5 was sent, it will
# cause md5 mismatches and retries as there was no indication that
# the stream being read from encountered any issues.
if self._transfer_coordinator.exception:
raise self._transfer_coordinator.exception
return self._fileobj.read(amount)
def seek(self, where, whence=0):
self._fileobj.seek(where, whence)
def tell(self):
return self._fileobj.tell()
def close(self):
self._fileobj.close()
def __enter__(self):
return self
def __exit__(self, *args, **kwargs):
self.close()
class UploadInputManager(object):
"""Base manager class for handling various types of files for uploads
This class is typically used for the UploadSubmissionTask class to help
determine the following:
* How to determine the size of the file
* How to determine if a multipart upload is required
* How to retrieve the body for a PutObject
* How to retrieve the bodies for a set of UploadParts
The answers/implementations differ for the various types of file inputs
that may be accepted. All implementations must subclass and override
public methods from this class.
"""
def __init__(self, osutil, transfer_coordinator, bandwidth_limiter=None):
self._osutil = osutil
self._transfer_coordinator = transfer_coordinator
self._bandwidth_limiter = bandwidth_limiter
@classmethod
def is_compatible(cls, upload_source):
"""Determines if the source for the upload is compatible with manager
:param upload_source: The source for which the upload will pull data
from.
:returns: True if the manager can handle the type of source specified
otherwise returns False.
"""
raise NotImplementedError('must implement _is_compatible()')
def stores_body_in_memory(self, operation_name):
"""Whether the body it provides are stored in-memory
:type operation_name: str
:param operation_name: The name of the client operation that the body
is being used for. Valid operation_names are ``put_object`` and
``upload_part``.
:rtype: boolean
:returns: True if the body returned by the manager will be stored in
memory. False if the manager will not directly store the body in
memory.
"""
raise NotImplemented('must implement store_body_in_memory()')
def provide_transfer_size(self, transfer_future):
"""Provides the transfer size of an upload
:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request
"""
raise NotImplementedError('must implement provide_transfer_size()')
def requires_multipart_upload(self, transfer_future, config):
"""Determines where a multipart upload is required
:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request
:type config: s3transfer.manager.TransferConfig
:param config: The config associated to the transfer manager
:rtype: boolean
:returns: True, if the upload should be multipart based on
configuartion and size. False, otherwise.
"""
raise NotImplementedError('must implement requires_multipart_upload()')
def get_put_object_body(self, transfer_future):
"""Returns the body to use for PutObject
:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request
:type config: s3transfer.manager.TransferConfig
:param config: The config associated to the transfer manager
:rtype: s3transfer.utils.ReadFileChunk
:returns: A ReadFileChunk including all progress callbacks
associated with the transfer future.
"""
raise NotImplementedError('must implement get_put_object_body()')
def yield_upload_part_bodies(self, transfer_future, chunksize):
"""Yields the part number and body to use for each UploadPart
:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request
:type chunksize: int
:param chunksize: The chunksize to use for this upload.
:rtype: int, s3transfer.utils.ReadFileChunk
:returns: Yields the part number and the ReadFileChunk including all
progress callbacks associated with the transfer future for that
specific yielded part.
"""
raise NotImplementedError('must implement yield_upload_part_bodies()')
def _wrap_fileobj(self, fileobj):
fileobj = InterruptReader(fileobj, self._transfer_coordinator)
if self._bandwidth_limiter:
fileobj = self._bandwidth_limiter.get_bandwith_limited_stream(
fileobj, self._transfer_coordinator, enabled=False)
return fileobj
def _get_progress_callbacks(self, transfer_future):
callbacks = get_callbacks(transfer_future, 'progress')
# We only want to be wrapping the callbacks if there are callbacks to
# invoke because we do not want to be doing any unnecessary work if
# there are no callbacks to invoke.
if callbacks:
return [AggregatedProgressCallback(callbacks)]
return []
def _get_close_callbacks(self, aggregated_progress_callbacks):
return [callback.flush for callback in aggregated_progress_callbacks]
class UploadFilenameInputManager(UploadInputManager):
"""Upload utility for filenames"""
@classmethod
def is_compatible(cls, upload_source):
return isinstance(upload_source, six.string_types)
def stores_body_in_memory(self, operation_name):
return False
def provide_transfer_size(self, transfer_future):
transfer_future.meta.provide_transfer_size(
self._osutil.get_file_size(
transfer_future.meta.call_args.fileobj))
def requires_multipart_upload(self, transfer_future, config):
return transfer_future.meta.size >= config.multipart_threshold
def get_put_object_body(self, transfer_future):
# Get a file-like object for the given input
fileobj, full_size = self._get_put_object_fileobj_with_full_size(
transfer_future)
# Wrap fileobj with interrupt reader that will quickly cancel
# uploads if needed instead of having to wait for the socket
# to completely read all of the data.
fileobj = self._wrap_fileobj(fileobj)
callbacks = self._get_progress_callbacks(transfer_future)
close_callbacks = self._get_close_callbacks(callbacks)
size = transfer_future.meta.size
# Return the file-like object wrapped into a ReadFileChunk to get
# progress.
return self._osutil.open_file_chunk_reader_from_fileobj(
fileobj=fileobj, chunk_size=size, full_file_size=full_size,
callbacks=callbacks, close_callbacks=close_callbacks)
def yield_upload_part_bodies(self, transfer_future, chunksize):
full_file_size = transfer_future.meta.size
num_parts = self._get_num_parts(transfer_future, chunksize)
for part_number in range(1, num_parts + 1):
callbacks = self._get_progress_callbacks(transfer_future)
close_callbacks = self._get_close_callbacks(callbacks)
start_byte = chunksize * (part_number - 1)
# Get a file-like object for that part and the size of the full
# file size for the associated file-like object for that part.
fileobj, full_size = self._get_upload_part_fileobj_with_full_size(
transfer_future.meta.call_args.fileobj, start_byte=start_byte,
part_size=chunksize, full_file_size=full_file_size)
# Wrap fileobj with interrupt reader that will quickly cancel
# uploads if needed instead of having to wait for the socket
# to completely read all of the data.
fileobj = self._wrap_fileobj(fileobj)
# Wrap the file-like object into a ReadFileChunk to get progress.
read_file_chunk = self._osutil.open_file_chunk_reader_from_fileobj(
fileobj=fileobj, chunk_size=chunksize,
full_file_size=full_size, callbacks=callbacks,
close_callbacks=close_callbacks)
yield part_number, read_file_chunk
def _get_deferred_open_file(self, fileobj, start_byte):
fileobj = DeferredOpenFile(
fileobj, start_byte, open_function=self._osutil.open)
return fileobj
def _get_put_object_fileobj_with_full_size(self, transfer_future):
fileobj = transfer_future.meta.call_args.fileobj
size = transfer_future.meta.size
return self._get_deferred_open_file(fileobj, 0), size
def _get_upload_part_fileobj_with_full_size(self, fileobj, **kwargs):
start_byte = kwargs['start_byte']
full_size = kwargs['full_file_size']
return self._get_deferred_open_file(fileobj, start_byte), full_size
def _get_num_parts(self, transfer_future, part_size):
return int(
math.ceil(transfer_future.meta.size / float(part_size)))
class UploadSeekableInputManager(UploadFilenameInputManager):
"""Upload utility for an open file object"""
@classmethod
def is_compatible(cls, upload_source):
return readable(upload_source) and seekable(upload_source)
def stores_body_in_memory(self, operation_name):
if operation_name == 'put_object':
return False
else:
return True
def provide_transfer_size(self, transfer_future):
fileobj = transfer_future.meta.call_args.fileobj
# To determine size, first determine the starting position
# Seek to the end and then find the difference in the length
# between the end and start positions.
start_position = fileobj.tell()
fileobj.seek(0, 2)
end_position = fileobj.tell()
fileobj.seek(start_position)
transfer_future.meta.provide_transfer_size(
end_position - start_position)
def _get_upload_part_fileobj_with_full_size(self, fileobj, **kwargs):
# Note: It is unfortunate that in order to do a multithreaded
# multipart upload we cannot simply copy the filelike object
# since there is not really a mechanism in python (i.e. os.dup
# points to the same OS filehandle which causes concurrency
# issues). So instead we need to read from the fileobj and
# chunk the data out to separate file-like objects in memory.
data = fileobj.read(kwargs['part_size'])
# We return the length of the data instead of the full_file_size
# because we partitioned the data into separate BytesIO objects
# meaning the BytesIO object has no knowledge of its start position
# relative the input source nor access to the rest of the input
# source. So we must treat it as its own standalone file.
return six.BytesIO(data), len(data)
def _get_put_object_fileobj_with_full_size(self, transfer_future):
fileobj = transfer_future.meta.call_args.fileobj
# The current position needs to be taken into account when retrieving
# the full size of the file.
size = fileobj.tell() + transfer_future.meta.size
return fileobj, size
class UploadNonSeekableInputManager(UploadInputManager):
"""Upload utility for a file-like object that cannot seek."""
def __init__(self, osutil, transfer_coordinator, bandwidth_limiter=None):
super(UploadNonSeekableInputManager, self).__init__(
osutil, transfer_coordinator, bandwidth_limiter)
self._initial_data = b''
@classmethod
def is_compatible(cls, upload_source):
return readable(upload_source)
def stores_body_in_memory(self, operation_name):
return True
def provide_transfer_size(self, transfer_future):
# No-op because there is no way to do this short of reading the entire
# body into memory.
return
def requires_multipart_upload(self, transfer_future, config):
# If the user has set the size, we can use that.
if transfer_future.meta.size is not None:
return transfer_future.meta.size >= config.multipart_threshold
# This is tricky to determine in this case because we can't know how
# large the input is. So to figure it out, we read data into memory
# up until the threshold and compare how much data was actually read
# against the threshold.
fileobj = transfer_future.meta.call_args.fileobj
threshold = config.multipart_threshold
self._initial_data = self._read(fileobj, threshold, False)
if len(self._initial_data) < threshold:
return False
else:
return True
def get_put_object_body(self, transfer_future):
callbacks = self._get_progress_callbacks(transfer_future)
close_callbacks = self._get_close_callbacks(callbacks)
fileobj = transfer_future.meta.call_args.fileobj
body = self._wrap_data(
self._initial_data + fileobj.read(), callbacks, close_callbacks)
# Zero out the stored data so we don't have additional copies
# hanging around in memory.
self._initial_data = None
return body
def yield_upload_part_bodies(self, transfer_future, chunksize):
file_object = transfer_future.meta.call_args.fileobj
part_number = 0
# Continue reading parts from the file-like object until it is empty.
while True:
callbacks = self._get_progress_callbacks(transfer_future)
close_callbacks = self._get_close_callbacks(callbacks)
part_number += 1
part_content = self._read(file_object, chunksize)
if not part_content:
break
part_object = self._wrap_data(
part_content, callbacks, close_callbacks)
# Zero out part_content to avoid hanging on to additional data.
part_content = None
yield part_number, part_object
def _read(self, fileobj, amount, truncate=True):
"""
Reads a specific amount of data from a stream and returns it. If there
is any data in initial_data, that will be popped out first.
:type fileobj: A file-like object that implements read
:param fileobj: The stream to read from.
:type amount: int
:param amount: The number of bytes to read from the stream.
:type truncate: bool
:param truncate: Whether or not to truncate initial_data after
reading from it.
:return: Generator which generates part bodies from the initial data.
"""
# If the the initial data is empty, we simply read from the fileobj
if len(self._initial_data) == 0:
return fileobj.read(amount)
# If the requested number of bytes is less than the amount of
# initial data, pull entirely from initial data.
if amount <= len(self._initial_data):
data = self._initial_data[:amount]
# Truncate initial data so we don't hang onto the data longer
# than we need.
if truncate:
self._initial_data = self._initial_data[amount:]
return data
# At this point there is some initial data left, but not enough to
# satisfy the number of bytes requested. Pull out the remaining
# initial data and read the rest from the fileobj.
amount_to_read = amount - len(self._initial_data)
data = self._initial_data + fileobj.read(amount_to_read)
# Zero out initial data so we don't hang onto the data any more.
if truncate:
self._initial_data = b''
return data
def _wrap_data(self, data, callbacks, close_callbacks):
"""
Wraps data with the interrupt reader and the file chunk reader.
:type data: bytes
:param data: The data to wrap.
:type callbacks: list
:param callbacks: The callbacks associated with the transfer future.
:type close_callbacks: list
:param close_callbacks: The callbacks to be called when closing the
wrapper for the data.
:return: Fully wrapped data.
"""
fileobj = self._wrap_fileobj(six.BytesIO(data))
return self._osutil.open_file_chunk_reader_from_fileobj(
fileobj=fileobj, chunk_size=len(data), full_file_size=len(data),
callbacks=callbacks, close_callbacks=close_callbacks)
class UploadSubmissionTask(SubmissionTask):
"""Task for submitting tasks to execute an upload"""
UPLOAD_PART_ARGS = [
'SSECustomerKey',
'SSECustomerAlgorithm',
'SSECustomerKeyMD5',
'RequestPayer',
]
COMPLETE_MULTIPART_ARGS = [
'RequestPayer'
]
def _get_upload_input_manager_cls(self, transfer_future):
"""Retrieves a class for managing input for an upload based on file type
:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The transfer future for the request
:rtype: class of UploadInputManager
:returns: The appropriate class to use for managing a specific type of
input for uploads.
"""
upload_manager_resolver_chain = [
UploadFilenameInputManager,
UploadSeekableInputManager,
UploadNonSeekableInputManager
]
fileobj = transfer_future.meta.call_args.fileobj
for upload_manager_cls in upload_manager_resolver_chain:
if upload_manager_cls.is_compatible(fileobj):
return upload_manager_cls
raise RuntimeError(
'Input %s of type: %s is not supported.' % (
fileobj, type(fileobj)))
def _submit(self, client, config, osutil, request_executor,
transfer_future, bandwidth_limiter=None):
"""
:param client: The client associated with the transfer manager
:type config: s3transfer.manager.TransferConfig
:param config: The transfer config associated with the transfer
manager
:type osutil: s3transfer.utils.OSUtil
:param osutil: The os utility associated to the transfer manager
:type request_executor: s3transfer.futures.BoundedExecutor
:param request_executor: The request executor associated with the
transfer manager
:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The transfer future associated with the
transfer request that tasks are being submitted for
"""
upload_input_manager = self._get_upload_input_manager_cls(
transfer_future)(
osutil, self._transfer_coordinator, bandwidth_limiter)
# Determine the size if it was not provided
if transfer_future.meta.size is None:
upload_input_manager.provide_transfer_size(transfer_future)
# Do a multipart upload if needed, otherwise do a regular put object.
if not upload_input_manager.requires_multipart_upload(
transfer_future, config):
self._submit_upload_request(
client, config, osutil, request_executor, transfer_future,
upload_input_manager)
else:
self._submit_multipart_request(
client, config, osutil, request_executor, transfer_future,
upload_input_manager)
def _submit_upload_request(self, client, config, osutil, request_executor,
transfer_future, upload_input_manager):
call_args = transfer_future.meta.call_args
# Get any tags that need to be associated to the put object task
put_object_tag = self._get_upload_task_tag(
upload_input_manager, 'put_object')
# Submit the request of a single upload.
self._transfer_coordinator.submit(
request_executor,
PutObjectTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'fileobj': upload_input_manager.get_put_object_body(
transfer_future),
'bucket': call_args.bucket,
'key': call_args.key,
'extra_args': call_args.extra_args
},
is_final=True
),
tag=put_object_tag
)
def _submit_multipart_request(self, client, config, osutil,
request_executor, transfer_future,
upload_input_manager):
call_args = transfer_future.meta.call_args
# Submit the request to create a multipart upload.
create_multipart_future = self._transfer_coordinator.submit(
request_executor,
CreateMultipartUploadTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'extra_args': call_args.extra_args,
}
)
)
# Submit requests to upload the parts of the file.
part_futures = []
extra_part_args = self._extra_upload_part_args(call_args.extra_args)
# Get any tags that need to be associated to the submitted task
# for upload the data
upload_part_tag = self._get_upload_task_tag(
upload_input_manager, 'upload_part')
size = transfer_future.meta.size
adjuster = ChunksizeAdjuster()
chunksize = adjuster.adjust_chunksize(config.multipart_chunksize, size)
part_iterator = upload_input_manager.yield_upload_part_bodies(
transfer_future, chunksize)
for part_number, fileobj in part_iterator:
part_futures.append(
self._transfer_coordinator.submit(
request_executor,
UploadPartTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'fileobj': fileobj,
'bucket': call_args.bucket,
'key': call_args.key,
'part_number': part_number,
'extra_args': extra_part_args
},
pending_main_kwargs={
'upload_id': create_multipart_future
}
),
tag=upload_part_tag
)
)
complete_multipart_extra_args = self._extra_complete_multipart_args(
call_args.extra_args)
# Submit the request to complete the multipart upload.
self._transfer_coordinator.submit(
request_executor,
CompleteMultipartUploadTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'extra_args': complete_multipart_extra_args,
},
pending_main_kwargs={
'upload_id': create_multipart_future,
'parts': part_futures
},
is_final=True
)
)
def _extra_upload_part_args(self, extra_args):
# Only the args in UPLOAD_PART_ARGS actually need to be passed
# onto the upload_part calls.
return get_filtered_dict(extra_args, self.UPLOAD_PART_ARGS)
def _extra_complete_multipart_args(self, extra_args):
return get_filtered_dict(extra_args, self.COMPLETE_MULTIPART_ARGS)
def _get_upload_task_tag(self, upload_input_manager, operation_name):
tag = None
if upload_input_manager.stores_body_in_memory(operation_name):
tag = IN_MEMORY_UPLOAD_TAG
return tag
class PutObjectTask(Task):
"""Task to do a nonmultipart upload"""
def _main(self, client, fileobj, bucket, key, extra_args):
"""
:param client: The client to use when calling PutObject
:param fileobj: The file to upload.
:param bucket: The name of the bucket to upload to
:param key: The name of the key to upload to
:param extra_args: A dictionary of any extra arguments that may be
used in the upload.
"""
with fileobj as body:
client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
class UploadPartTask(Task):
"""Task to upload a part in a multipart upload"""
def _main(self, client, fileobj, bucket, key, upload_id, part_number,
extra_args):
"""
:param client: The client to use when calling PutObject
:param fileobj: The file to upload.
:param bucket: The name of the bucket to upload to
:param key: The name of the key to upload to
:param upload_id: The id of the upload
:param part_number: The number representing the part of the multipart
upload
:param extra_args: A dictionary of any extra arguments that may be
used in the upload.
:rtype: dict
:returns: A dictionary representing a part::
{'Etag': etag_value, 'PartNumber': part_number}
This value can be appended to a list to be used to complete
the multipart upload.
"""
with fileobj as body:
response = client.upload_part(
Bucket=bucket, Key=key,
UploadId=upload_id, PartNumber=part_number,
Body=body, **extra_args)
etag = response['ETag']
return {'ETag': etag, 'PartNumber': part_number}
Zerion Mini Shell 1.0