Skip to content
Permalink
Browse files

s3: support server-side encryption

The S3 API supports server-side encryption of S3 buckets. For users
already managing encryption server-side it is convenient to do the
same for Postgres backups rather than use a separate client-side
mechanism.
  • Loading branch information...
Dave Peticolas
Dave Peticolas committed Aug 30, 2016
1 parent e992a78 commit a1938e8f3f0bae1d11fad542635dcebb3094cca7
Showing with 40 additions and 8 deletions.
  1. +1 −0 .gitignore
  2. +4 −0 README.rst
  3. +12 −5 pghoard/rohmu/object_storage/s3.py
  4. +23 −3 test/test_storage.py
@@ -1,6 +1,7 @@
*.pyc
*.pyo
*~
/.cache
/.coverage
/.project
/.pydevproject
@@ -487,6 +487,10 @@ The following object storage types are supported:
* ``region`` S3 region of the bucket
* ``bucket_name`` name of the S3 bucket

Optional keys for Amazon Web Services S3:

* ``encrypted`` if True, use server-side encryption. Default is False.

* ``s3`` for other S3 compatible services such as Ceph, required
configuration keys:

@@ -38,7 +38,8 @@ def __init__(self,
host=None,
port=None,
is_secure=False,
segment_size=MULTIPART_CHUNK_SIZE):
segment_size=MULTIPART_CHUNK_SIZE,
encrypted=False):
super().__init__(prefix=prefix)
self.region = region
self.location = _location_for_region(region)
@@ -53,6 +54,7 @@ def __init__(self,
aws_secret_access_key=aws_secret_access_key)
self.bucket = self.get_or_create_bucket(self.bucket_name)
self.multipart_chunk_size = segment_size
self.encrypted = encrypted
self.log.debug("S3Transfer initialized")

def get_metadata_for_key(self, key):
@@ -81,10 +83,13 @@ def list_path(self, key):
for item in self.bucket.list(path, "/"):
if not hasattr(item, "last_modified"):
continue # skip objects with no last_modified: not regular objects
name = self.format_key_from_backend(item.name)
if name == path:
continue # skip the path itself
result.append({
"last_modified": dateutil.parser.parse(item.last_modified),
"metadata": self._metadata_for_key(item.name),
"name": self.format_key_from_backend(item.name),
"name": name,
"size": item.size,
})
return result
@@ -118,7 +123,7 @@ def store_file_from_memory(self, key, memstring, metadata=None):
if metadata:
for k, v in self.sanitize_metadata(metadata).items():
s3key.set_metadata(k, v)
s3key.set_contents_from_string(memstring, replace=True)
s3key.set_contents_from_string(memstring, replace=True, encrypt_key=self.encrypted)

def _store_multipart_upload(self, mp, fp, part_num, filepath):
attempt = 0
@@ -153,13 +158,15 @@ def store_file_from_disk(self, key, filepath, metadata=None, multipart=None):
if metadata:
for k, v in metadata.items():
s3key.set_metadata(k, v)
s3key.set_contents_from_filename(filepath, replace=True)
s3key.set_contents_from_filename(filepath, replace=True,
encrypt_key=self.encrypted)
else:
start_of_multipart_upload = time.monotonic()
chunks = math.ceil(size / self.multipart_chunk_size)
self.log.debug("Starting to upload multipart file: %r, size: %r, chunks: %d",
key, size, chunks)
mp = self.bucket.initiate_multipart_upload(key, metadata=metadata)
mp = self.bucket.initiate_multipart_upload(key, metadata=metadata,
encrypt_key=self.encrypted)

with open(filepath, "rb") as fp:
part_num = 0
@@ -19,7 +19,7 @@
test_storage_configs = object()


def _test_storage(st, driver, tmpdir):
def _test_storage(st, driver, tmpdir, storage_config):
scratch = tmpdir.join("scratch")
compat.makedirs(str(scratch), exist_ok=True)

@@ -55,6 +55,10 @@ def _test_storage(st, driver, tmpdir):
assert st.get_contents_to_fileobj("test1/x1", out) == {}
assert out.getvalue() == b"from disk"

if driver == "s3":
key = st.bucket.get_key(st.format_key_for_backend("test1/x1"))
assert bool(key.encrypted) == bool(storage_config.get('encrypted'))

st.store_file_from_memory("test1/x1", b"dummy", {"k": "v"})
out = BytesIO()
assert st.get_contents_to_fileobj("test1/x1", out) == {"k": "v"}
@@ -70,6 +74,10 @@ def _test_storage(st, driver, tmpdir):
to_disk_file = str(scratch.join("b"))
assert st.get_contents_to_file("test1/td", to_disk_file) == {"to-disk": "42"}

if driver == "s3":
key = st.bucket.get_key(st.format_key_for_backend("test1/td"))
assert bool(key.encrypted) == bool(storage_config.get('encrypted'))

assert st.list_path("") == [] # nothing at top level (directories not listed)
if driver == "local":
# create a dot-file (hidden), this must be ignored
@@ -144,6 +152,9 @@ def failing_new_key(key_name): # pylint: disable=unused-argument
metadata={"thirtymeg": "data", "size": test_size_send, "key": "value-with-a-hyphen"})

assert fail_calls[0] > 3

key = st.bucket.get_key(st.format_key_for_backend("test1/30m"))
assert bool(key.encrypted) == bool(storage_config.get('encrypted'))
else:
st.store_file_from_disk("test1/30m", test_file, multipart=True,
metadata={"thirtymeg": "data", "size": test_size_send, "key": "value-with-a-hyphen"})
@@ -206,7 +217,7 @@ def dl_progress(current_pos, expected_max):
assert st.list_path("test1_segments/30m") == []


def _test_storage_init(storage_type, with_prefix, tmpdir):
def _test_storage_init(storage_type, with_prefix, tmpdir, config_overrides=None):
if storage_type == "local":
storage_config = {"directory": str(tmpdir.join("rohmu"))}
else:
@@ -227,8 +238,12 @@ def _test_storage_init(storage_type, with_prefix, tmpdir):
if with_prefix:
storage_config["prefix"] = uuid.uuid4().hex

if config_overrides:
storage_config = storage_config.copy()
storage_config.update(config_overrides)

st = get_transfer(storage_config)
_test_storage(st, driver, tmpdir)
_test_storage(st, driver, tmpdir, storage_config)


def test_storage_aws_s3_no_prefix(tmpdir):
@@ -239,6 +254,11 @@ def test_storage_aws_s3_with_prefix(tmpdir):
_test_storage_init("aws_s3", True, tmpdir)


def test_storage_aws_s3_no_prefix_with_encryption(tmpdir):
_test_storage_init("aws_s3", False, tmpdir,
config_overrides={'encrypted': True})


def test_storage_azure_no_prefix(tmpdir):
_test_storage_init("azure", False, tmpdir)

0 comments on commit a1938e8

Please sign in to comment.
You can’t perform that action at this time.