Permalink
Browse files

Merge pull request #3799 from houglum/develop

Support fetching GCS bucket encryption metadata.
  • Loading branch information...
mfschwartz committed Mar 8, 2018
2 parents 5334015 + c4dbe6f commit 132b64d21e071bf6cc4ccfe5ec9cb4273edeba2f
@@ -475,9 +475,12 @@ def __init__(self, message):
self.message = message


class NoAuthHandlerFound(Exception):
"""Is raised when no auth handlers were found ready to authenticate."""
pass
class InvalidEncryptionConfigError(Exception):
"""Exception raised when GCS encryption configuration XML is invalid."""

def __init__(self, message):
super(InvalidEncryptionConfigError, self).__init__(message)
self.message = message


class InvalidLifecycleConfigError(Exception):
@@ -488,6 +491,11 @@ def __init__(self, message):
self.message = message


class NoAuthHandlerFound(Exception):
"""Is raised when no auth handlers were found ready to authenticate."""
pass


# Enum class for resumable upload failure disposition.
class ResumableTransferDisposition(object):
# START_OVER means an attempt to resume an existing transfer failed,
@@ -32,6 +32,7 @@
from boto.gs.acl import SupportedPermissions as GSPermissions
from boto.gs.bucketlistresultset import VersionedBucketListResultSet
from boto.gs.cors import Cors
from boto.gs.encryptionconfig import EncryptionConfig
from boto.gs.lifecycle import LifecycleConfig
from boto.gs.key import Key as GSKey
from boto.s3.acl import Policy
@@ -43,6 +44,7 @@
DEF_OBJ_ACL = 'defaultObjectAcl'
STANDARD_ACL = 'acl'
CORS_ARG = 'cors'
ENCRYPTION_CONFIG_ARG = 'encryptionConfig'
LIFECYCLE_ARG = 'lifecycle'
STORAGE_CLASS_ARG='storageClass'
ERROR_DETAILS_REGEX = re.compile(r'<Details>(?P<details>.*)</Details>')
@@ -51,12 +53,19 @@ class Bucket(S3Bucket):
"""Represents a Google Cloud Storage bucket."""

BillingBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<BillingConfiguration><RequesterPays>%s</RequesterPays>'
'<BillingConfiguration>'
'<RequesterPays>%s</RequesterPays>'
'</BillingConfiguration>')
EncryptionConfigBody = (
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<EncryptionConfiguration>%s</EncryptionConfiguration>')
EncryptionConfigDefaultKeyNameFragment = (
'<DefaultKmsKeyName>%s</DefaultKmsKeyName>')
StorageClassBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<StorageClass>%s</StorageClass>')
VersioningBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<VersioningConfiguration><Status>%s</Status>'
'<VersioningConfiguration>'
'<Status>%s</Status>'
'</VersioningConfiguration>')
WebsiteBody = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<WebsiteConfiguration>%s%s</WebsiteConfiguration>')
@@ -1065,3 +1074,61 @@ def configure_billing(self, requester_pays=False, headers=None):
else:
req_body = self.BillingBody % ('Disabled')
self.set_subresource('billing', req_body, headers=headers)

def get_encryption_config(self, headers=None):
"""Returns a bucket's EncryptionConfig.
:param dict headers: Additional headers to send with the request.
:rtype: :class:`~.encryption_config.EncryptionConfig`
"""
response = self.connection.make_request(
'GET', self.name, query_args=ENCRYPTION_CONFIG_ARG, headers=headers)
body = response.read()
if response.status == 200:
# Success - parse XML and return EncryptionConfig object.
encryption_config = EncryptionConfig()
h = handler.XmlHandler(encryption_config, self)
xml.sax.parseString(body, h)
return encryption_config
else:
raise self.connection.provider.storage_response_error(
response.status, response.reason, body)

def _construct_encryption_config_xml(self, default_kms_key_name=None):
"""Creates an XML document for setting a bucket's EncryptionConfig.
This method is internal as it's only here for testing purposes. As
managing Cloud KMS resources for testing is complex, we settle for
testing that we're creating correctly-formed XML for setting a bucket's
encryption configuration.
:param str default_kms_key_name: A string containing a fully-qualified
Cloud KMS key name.
:rtype: str
"""
if default_kms_key_name:
default_kms_key_name_frag = (
self.EncryptionConfigDefaultKeyNameFragment %
default_kms_key_name)
else:
default_kms_key_name_frag = ''

return self.EncryptionConfigBody % default_kms_key_name_frag


def set_encryption_config(self, default_kms_key_name=None, headers=None):
"""Sets a bucket's EncryptionConfig XML document.
:param str default_kms_key_name: A string containing a fully-qualified
Cloud KMS key name.
:param dict headers: Additional headers to send with the request.
"""
body = self._construct_encryption_config_xml(
default_kms_key_name=default_kms_key_name)
response = self.connection.make_request(
'PUT', get_utf8_value(self.name), data=get_utf8_value(body),
query_args=ENCRYPTION_CONFIG_ARG, headers=headers)
body = response.read()
if response.status != 200:
raise self.connection.provider.storage_response_error(
response.status, response.reason, body)
@@ -0,0 +1,76 @@
# Copyright 2018 Google Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import types
from boto.gs.user import User
from boto.exception import InvalidEncryptionConfigError
from xml.sax import handler

# Relevant tags for the EncryptionConfiguration XML document.
DEFAULT_KMS_KEY_NAME = 'DefaultKmsKeyName'
ENCRYPTION_CONFIG = 'EncryptionConfiguration'

class EncryptionConfig(handler.ContentHandler):
"""Encapsulates the EncryptionConfiguration XML document"""
def __init__(self):
# Valid items in an EncryptionConfiguration XML node.
self.default_kms_key_name = None

self.parse_level = 0

def validateParseLevel(self, tag, level):
"""Verify parse level for a given tag."""
if self.parse_level != level:
raise InvalidEncryptionConfigError(
'Invalid tag %s at parse level %d: ' % (tag, self.parse_level))

def startElement(self, name, attrs, connection):
"""SAX XML logic for parsing new element found."""
if name == ENCRYPTION_CONFIG:
self.validateParseLevel(name, 0)
self.parse_level += 1;
elif name == DEFAULT_KMS_KEY_NAME:
self.validateParseLevel(name, 1)
self.parse_level += 1;
else:
raise InvalidEncryptionConfigError('Unsupported tag ' + name)

def endElement(self, name, value, connection):
"""SAX XML logic for parsing new element found."""
if name == ENCRYPTION_CONFIG:
self.validateParseLevel(name, 1)
self.parse_level -= 1;
elif name == DEFAULT_KMS_KEY_NAME:
self.validateParseLevel(name, 2)
self.parse_level -= 1;
self.default_kms_key_name = value.strip()
else:
raise InvalidEncryptionConfigError('Unsupported end tag ' + name)

def to_xml(self):
"""Convert EncryptionConfig object into XML string representation."""
s = ['<%s>' % ENCRYPTION_CONFIG]
if self.default_kms_key_name:
s.append('<%s>%s</%s>' % (DEFAULT_KMS_KEY_NAME,
self.default_kms_key_name,
DEFAULT_KMS_KEY_NAME))
s.append('</%s>' % ENCRYPTION_CONFIG)
return ''.join(s)
@@ -821,6 +821,25 @@ def configure_billing(self, requester_pays=False, validate=False,
bucket = self.get_bucket(validate, headers)
bucket.configure_billing(requester_pays=requester_pays, headers=headers)

def get_encryption_config(self, validate=False, headers=None):
"""Returns a GCS bucket's encryption configuration."""
self._check_bucket_uri('get_encryption_config')
# EncryptionConfiguration is defined as a bucket param for GCS, but not
# for S3.
if self.scheme != 'gs':
raise ValueError('get_encryption_config() not supported for %s '
'URIs.' % self.scheme)
bucket = self.get_bucket(validate, headers)
return bucket.get_encryption_config(headers=headers)

def set_encryption_config(self, default_kms_key_name=None, validate=False,
headers=None):
"""Sets a GCS bucket's encryption configuration."""
self._check_bucket_uri('set_encryption_config')
bucket = self.get_bucket(validate, headers)
bucket.set_encryption_config(default_kms_key_name=default_kms_key_name,
headers=headers)

def exists(self, headers=None):
"""Returns True if the object exists or False if it doesn't"""
if not self.object_name:
@@ -93,7 +93,10 @@
# billing is a QSA for buckets in Google Cloud Storage.
'billing',
# userProject is a QSA for requests in Google Cloud Storage.
'userProject']
'userProject',
# encryptionConfig is a QSA for requests in Google Cloud
# Storage.
'encryptionConfig']


_first_cap_regex = re.compile('(.)([A-Z][a-z]+)')
@@ -51,6 +51,12 @@
'<ResponseHeader>bar</ResponseHeader></ResponseHeaders>'
'</Cors></CorsConfig>')

ENCRYPTION_CONFIG_WITH_KEY = (
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<EncryptionConfiguration>'
'<DefaultKmsKeyName>%s</DefaultKmsKeyName>'
'</EncryptionConfiguration>')

LIFECYCLE_EMPTY = ('<?xml version="1.0" encoding="UTF-8"?>'
'<LifecycleConfiguration></LifecycleConfiguration>')
LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>'
@@ -491,3 +497,34 @@ def test_billing_config_storage_uri(self):
uri.configure_billing(requester_pays=False)
billing = uri.get_billing_config()
self.assertEqual(billing, BILLING_DISABLED)

def test_encryption_config_bucket(self):
"""Test setting and getting of EncryptionConfig on gs Bucket objects."""
# Create a new bucket.
bucket = self._MakeBucket()
bucket_name = bucket.name
# Get EncryptionConfig and make sure it's empty.
encryption_config = bucket.get_encryption_config()
self.assertIsNone(encryption_config.default_kms_key_name)
# Testing set functionality would require having an existing Cloud KMS
# key. Since we can't hardcode a key name or dynamically create one, we
# only test here that we're creating the correct XML document to send to
# GCS.
xmldoc = bucket._construct_encryption_config_xml(
default_kms_key_name='dummykey')
self.assertEqual(xmldoc, ENCRYPTION_CONFIG_WITH_KEY % 'dummykey')
# Test that setting an empty encryption config works.
bucket.set_encryption_config()

def test_encryption_config_storage_uri(self):
"""Test setting and getting of EncryptionConfig with storage_uri."""
# Create a new bucket.
bucket = self._MakeBucket()
bucket_name = bucket.name
uri = storage_uri('gs://' + bucket_name)
# Get EncryptionConfig and make sure it's empty.
encryption_config = uri.get_encryption_config()
self.assertIsNone(encryption_config.default_kms_key_name)

# Test that setting an empty encryption config works.
uri.set_encryption_config()
@@ -423,7 +423,8 @@ def test_header_encoding(self):
check.cache_control,
('public,%20max-age=500', 'public, max-age=500')
)
self.assertEqual(remote_metadata['cache-control'], 'public,%20max-age=500')
self.assertIn(remote_metadata['cache-control'],
('public,%20max-age=500', 'public, max-age=500'))
self.assertEqual(check.get_metadata('test-plus'), 'A plus (+)')
self.assertEqual(check.content_disposition, 'filename=Sch%C3%B6ne%20Zeit.txt')
self.assertEqual(remote_metadata['content-disposition'], 'filename=Sch%C3%B6ne%20Zeit.txt')

0 comments on commit 132b64d

Please sign in to comment.