Expand Up @@ -14,7 +14,9 @@
"""Create / interact with Google Cloud Storage blobs."""
import base64 import copy import hashlib from io import BytesIO import json import mimetypes Expand All @@ -26,6 +28,8 @@ from six.moves.urllib.parse import quote
from gcloud._helpers import _rfc3339_to_datetime from gcloud._helpers import _to_bytes from gcloud._helpers import _bytes_to_unicode from gcloud.credentials import generate_signed_url from gcloud.exceptions import NotFound from gcloud.exceptions import make_exception Expand Down Expand Up @@ -276,17 +280,41 @@ def delete(self, client=None): """ return self.bucket.delete_blob(self.name, client=client)
def download_to_file(self, file_obj, client=None): def download_to_file(self, file_obj, encryption_key=None, client=None): """Download the contents of this blob into a file-like object.
.. note::
If the server-set property, :attr:`media_link`, is not yet initialized, makes an additional API request to load it.
Downloading a file that has been encrypted with a `customer-supplied`_ encryption key::
>>> from gcloud import storage >>> from gcloud.storage import Blob
>>> client = storage.Client(project='my-project') >>> bucket = client.get_bucket('my-bucket') >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' >>> blob = Blob('secure-data', bucket) >>> with open('/tmp/my-secure-file', 'wb') as file_obj: >>> blob.download_to_file(file_obj, ... encryption_key=encryption_key)
The ``encryption_key`` should be a str or bytes with a length of at least 32.
.. _customer-supplied: https://cloud.google.com/storage/docs/\ encryption#customer-supplied
:type file_obj: file :param file_obj: A file handle to which to write the blob's data.
:type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption.
:type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. Expand All @@ -305,7 +333,11 @@ def download_to_file(self, file_obj, client=None): if self.chunk_size is not None: download.chunksize = self.chunk_size
request = Request(download_url, 'GET') headers = {} if encryption_key: _set_encryption_headers(encryption_key, headers)
request = Request(download_url, 'GET', headers)
# Use the private ``_connection`` rather than the public # ``.connection``, since the public connection may be a batch. A Expand All @@ -315,27 +347,36 @@ def download_to_file(self, file_obj, client=None): # it has all three (http, API_BASE_URL and build_api_url). download.initialize_download(request, client._connection.http)
def download_to_filename(self, filename, client=None): def download_to_filename(self, filename, encryption_key=None, client=None): """Download the contents of this blob into a named file.
:type filename: string :param filename: A filename to be passed to ``open``.
:type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption.
:type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket.
:raises: :class:`gcloud.exceptions.NotFound` """ with open(filename, 'wb') as file_obj: self.download_to_file(file_obj, client=client) self.download_to_file(file_obj, encryption_key=encryption_key, client=client)
mtime = time.mktime(self.updated.timetuple()) os.utime(file_obj.name, (mtime, mtime))
def download_as_string(self, client=None): def download_as_string(self, encryption_key=None, client=None): """Download the contents of this blob as a string.
:type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption.
:type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. Expand All @@ -345,7 +386,8 @@ def download_as_string(self, client=None): :raises: :class:`gcloud.exceptions.NotFound` """ string_buffer = BytesIO() self.download_to_file(string_buffer, client=client) self.download_to_file(string_buffer, encryption_key=encryption_key, client=client) return string_buffer.getvalue()
@staticmethod Expand All @@ -358,8 +400,10 @@ def _check_response_error(request, http_response): raise make_exception(faux_response, http_response.content, error_info=request.url)
# pylint: disable=too-many-locals def upload_from_file(self, file_obj, rewind=False, size=None, content_type=None, num_retries=6, client=None): encryption_key=None, content_type=None, num_retries=6, client=None): """Upload the contents of this blob from a file-like object.
The content type of the upload will either be Expand All @@ -378,6 +422,25 @@ def upload_from_file(self, file_obj, rewind=False, size=None, `lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_ API documents for details.
Uploading a file with a `customer-supplied`_ encryption key::
>>> from gcloud import storage >>> from gcloud.storage import Blob
>>> client = storage.Client(project='my-project') >>> bucket = client.get_bucket('my-bucket') >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' >>> blob = Blob('secure-data', bucket) >>> with open('my-file', 'rb') as my_file: >>> blob.upload_from_file(my_file, ... encryption_key=encryption_key)
The ``encryption_key`` should be a str or bytes with a length of at least 32.
.. _customer-supplied: https://cloud.google.com/storage/docs/\ encryption#customer-supplied
:type file_obj: file :param file_obj: A file handle open for reading.
Expand All @@ -391,6 +454,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None, :func:`os.fstat`. (If the file handle is not from the filesystem this won't be possible.)
:type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption.
:type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded.
Expand Down Expand Up @@ -434,6 +501,9 @@ def upload_from_file(self, file_obj, rewind=False, size=None, 'User-Agent': connection.USER_AGENT, }
if encryption_key: _set_encryption_headers(encryption_key, headers)
upload = Upload(file_obj, content_type, total_bytes, auto_transfer=False)
Expand Down Expand Up @@ -473,9 +543,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None, six.string_types): # pragma: NO COVER Python3 response_content = response_content.decode('utf-8') self._set_properties(json.loads(response_content)) # pylint: enable=too-many-locals
def upload_from_filename(self, filename, content_type=None, client=None): encryption_key=None, client=None): """Upload this blob's contents from the content of a named file.
The content type of the upload will either be Expand All @@ -500,6 +571,10 @@ def upload_from_filename(self, filename, content_type=None, :type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded.
:type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption.
:type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. Expand All @@ -510,10 +585,10 @@ def upload_from_filename(self, filename, content_type=None,
with open(filename, 'rb') as file_obj: self.upload_from_file(file_obj, content_type=content_type, client=client) encryption_key=encryption_key, client=client)
def upload_from_string(self, data, content_type='text/plain', client=None): encryption_key=None, client=None): """Upload contents of this blob from the provided string.
.. note:: Expand All @@ -535,6 +610,10 @@ def upload_from_string(self, data, content_type='text/plain', :param content_type: Optional type of content being uploaded. Defaults to ``'text/plain'``.
:type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption.
:type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. Expand All @@ -545,7 +624,7 @@ def upload_from_string(self, data, content_type='text/plain', string_buffer.write(data) self.upload_from_file(file_obj=string_buffer, rewind=True, size=len(data), content_type=content_type, client=client) encryption_key=encryption_key, client=client)
def make_public(self, client=None): """Make this blob public giving all users read access. Expand Down Expand Up @@ -838,3 +917,21 @@ def __init__(self, bucket_name, object_name): self.query_params = {'name': object_name} self._bucket_name = bucket_name self._relative_path = ''

def _set_encryption_headers(key, headers): """Builds customer encyrption key headers
:type key: str or bytes :param key: 32 byte key to build request key and hash.
:type headers: dict :param headers: dict of HTTP headers being sent in request. """ key = _to_bytes(key) sha256_key = hashlib.sha256(key).digest() key_hash = base64.b64encode(sha256_key).rstrip() encoded_key = base64.b64encode(key).rstrip() headers['X-Goog-Encryption-Algorithm'] = 'AES256' headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key) headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash)