Skip to content

Commit

Permalink
Merge 1fa7ffa into 909ef10
Browse files Browse the repository at this point in the history
  • Loading branch information
@hangzws
hangzws committed Oct 11, 2019
2 parents 909ef10 + 1fa7ffa commit 818652a
Show file tree
Hide file tree
Showing 38 changed files with 3,655 additions and 1,562 deletions.
105 changes: 91 additions & 14 deletions examples/custom_crypto.py
View file
Expand Up @@ -5,6 +5,7 @@
import oss2
from oss2.crypto import BaseCryptoProvider
from oss2.utils import b64encode_as_string, b64decode_from_string, to_bytes
from oss2.headers import *

from Crypto.Cipher import PKCS1_OAEP
from Crypto.PublicKey import RSA
Expand Down Expand Up @@ -33,10 +34,10 @@ def get_key():
return 'fake_key'

@staticmethod
def get_start():
def get_iv():
return 'fake_start'

def __init__(self, key=None, start=None):
def __init__(self, key=None, start=None, count=None):
pass

def encrypt(self, raw):
Expand Down Expand Up @@ -74,42 +75,70 @@ def __init__(self, cipher=FakeCrypto):
self.private_key = self.public_key


def build_header(self, headers=None):
def build_header(self, headers=None, multipart_context=None):
if not isinstance(headers, CaseInsensitiveDict):
headers = CaseInsensitiveDict(headers)

if 'content-md5' in headers:
headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5']
headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5']
del headers['content-md5']

if 'content-length' in headers:
headers['x-oss-meta-unencrypted-content-length'] = headers['content-length']
headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length']
del headers['content-length']

headers['x-oss-meta-oss-crypto-key'] = b64encode_as_string(self.public_key.encrypt(self.plain_key))
headers['x-oss-meta-oss-crypto-start'] = b64encode_as_string(self.public_key.encrypt(to_bytes(str(self.plain_start))))
headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM
headers['x-oss-meta-oss-wrap-alg'] = 'custom'
headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = b64encode_as_string(self.public_key.encrypt(self.plain_key))
headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = b64encode_as_string(self.public_key.encrypt(to_bytes(str(self.plain_iv))))
headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM
headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'custom'

# multipart file build header
if multipart_context:
headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size)
headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size)

self.plain_key = None
self.plain_start = None
self.plain_iv = None

return headers

def build_header_for_upload_part(self, headers=None):
if not isinstance(headers, CaseInsensitiveDict):
headers = CaseInsensitiveDict(headers)

if 'content-md5' in headers:
headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5']
del headers['content-md5']

if 'content-length' in headers:
headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length']
del headers['content-length']

self.plain_key = None
self.plain_iv = None

return headers

def get_key(self):
self.plain_key = self.cipher.get_key()
return self.plain_key

def get_start(self):
self.plain_start = self.cipher.get_start()
return self.plain_start
def get_iv(self):
self.plain_iv = self.cipher.get_iv()
return self.plain_iv

def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x):
try:
return conv(self.private_key.decrypt(b64decode_from_string(headers[key])))
except:
return None

def decrypt_from_str(self, key, value, conv=lambda x:x):
try:
return conv(self.private_key.decrypt(b64decode_from_string(value)))
except:
return None



# 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。
Expand Down Expand Up @@ -162,4 +191,52 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x):
with open(filename, 'rb') as fileobj:
assert fileobj.read() == content

os.remove(filename)
os.remove(filename)

"""
分片上传
"""
# 初始化上传分片
part_a = b'a' * 1024 * 100
part_b = b'b' * 1024 * 100
part_c = b'c' * 1024 * 100
multi_content = [part_a, part_b, part_c]

parts = []
data_size = 100 * 1024 * 3
part_size = 100 * 1024
multi_key = "test_crypto_multipart"

res = bucket.init_multipart_upload(multi_key, data_size, part_size)
upload_id = res.upload_id
crypto_multipart_context = res.crypto_multipart_context;

# 分片上传
for i in range(3):
result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context)
parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc))

## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。
#for i in range(2):
# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context)
# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc))
#
#res = bucket.list_parts(multi_key, upload_id)
#crypto_multipart_context_new = res.crypto_multipart_context
#
#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new)
#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc))

# 完成上传
result = bucket.complete_multipart_upload(multi_key, upload_id, parts)

# 下载全部文件
result = bucket.get_object(multi_key)

# 验证一下
content_got = b''
for chunk in result:
content_got += chunk
assert content_got[0:102400] == part_a
assert content_got[102400:204800] == part_b
assert content_got[204800:307200] == part_c
151 changes: 133 additions & 18 deletions examples/object_crypto.py
View file
@@ -1,11 +1,17 @@
# -*- coding: utf-8 -*-

import os
import sys
from Crypto.PublicKey import RSA
from Crypto.PublicKey.RSA import RsaKey

sys.path.append("/Users/fengyu/aliyun-oss-python-sdk")

import oss2
from oss2.crypto import LocalRsaProvider, AliKMSProvider
from oss2 import LocalRsaProvider, AliKMSProvider, RsaProvider
from oss2 import models

# 以下代码展示了客户端文件加密上传下载的用法，如下载文件、上传文件等，注意在客户端加密的条件下，oss暂不支持文件分片上传下载操作。
# 以下代码展示了客户端文件加密上传下载的用法，如下载文件、上传文件等。


# 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。
Expand All @@ -15,12 +21,12 @@
# http://oss-cn-hangzhou.aliyuncs.com
# https://oss-cn-hangzhou.aliyuncs.com
# 分别以HTTP、HTTPS协议访问。
access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '<你的AccessKeyId>')
access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '<你的AccessKeySecret>')
bucket_name = os.getenv('OSS_TEST_BUCKET', '<你的Bucket>')
endpoint = os.getenv('OSS_TEST_ENDPOINT', '<你的访问域名>')
cmk = os.getenv('OSS_TEST_CMK', '<你的CMK>')
region = os.getenv('OSS_TEST_REGION', '<你的区域>')
access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '')
access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '')
bucket_name = os.getenv('OSS_TEST_BUCKET', '')
endpoint = os.getenv('OSS_TEST_ENDPOINT', '')
cmk = os.getenv('OSS_TEST_CMK', '')
region = os.getenv('OSS_TEST_REGION', '')

# 确认上面的参数都填写正确了
for param in (access_key_id, access_key_secret, bucket_name, endpoint, cmk, region):
Expand All @@ -30,11 +36,32 @@
content = b'a' * 1024 * 1024
filename = 'download.txt'


# 创建Bucket对象，可以进行客户端数据加密(用户端RSA)，此模式下只提供对象整体上传下载操作
bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, crypto_provider=LocalRsaProvider())

key1 = 'motto-copy.txt'
private_key = '''-----BEGIN RSA PRIVATE KEY-----
MIICWwIBAAKBgQCokfiAVXXf5ImFzKDw+XO/UByW6mse2QsIgz3ZwBtMNu59fR5z
ttSx+8fB7vR4CN3bTztrP9A6bjoN0FFnhlQ3vNJC5MFO1PByrE/MNd5AAfSVba93
I6sx8NSk5MzUCA4NJzAUqYOEWGtGBcom6kEF6MmR1EKib1Id8hpooY5xaQIDAQAB
AoGAOPUZgkNeEMinrw31U3b2JS5sepG6oDG2CKpPu8OtdZMaAkzEfVTJiVoJpP2Y
nPZiADhFW3e0ZAnak9BPsSsySRaSNmR465cG9tbqpXFKh9Rp/sCPo4Jq2n65yood
JBrnGr6/xhYvNa14sQ6xjjfSgRNBSXD1XXNF4kALwgZyCAECQQDV7t4bTx9FbEs5
36nAxPsPM6aACXaOkv6d9LXI7A0J8Zf42FeBV6RK0q7QG5iNNd1WJHSXIITUizVF
6aX5NnvFAkEAybeXNOwUvYtkgxF4s28s6gn11c5HZw4/a8vZm2tXXK/QfTQrJVXp
VwxmSr0FAajWAlcYN/fGkX1pWA041CKFVQJAG08ozzekeEpAuByTIOaEXgZr5MBQ
gBbHpgZNBl8Lsw9CJSQI15wGfv6yDiLXsH8FyC9TKs+d5Tv4Cvquk0efOQJAd9OC
lCKFs48hdyaiz9yEDsc57PdrvRFepVdj/gpGzD14mVerJbOiOF6aSV19ot27u4on
Td/3aifYs0CveHzFPQJAWb4LCDwqLctfzziG7/S7Z74gyq5qZF4FUElOAZkz718E
yZvADwuz/4aK0od0lX9c4Jp7Mo5vQ4TvdoBnPuGoyw==
-----END RSA PRIVATE KEY-----'''

public_key = '''-----BEGIN RSA PUBLIC KEY-----
MIGJAoGBAKiR+IBVdd/kiYXMoPD5c79QHJbqax7ZCwiDPdnAG0w27n19HnO21LH7
x8Hu9HgI3dtPO2s/0DpuOg3QUWeGVDe80kLkwU7U8HKsT8w13kAB9JVtr3cjqzHw
1KTkzNQIDg0nMBSpg4RYa0YFyibqQQXoyZHUQqJvUh3yGmihjnFpAgMBAAE=
-----END RSA PUBLIC KEY-----'''


key_pair = {'private_key': private_key, 'public_key': public_key}
bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name,
crypto_provider=RsaProvider(key_pair))

# 上传文件
bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)})
Expand All @@ -51,8 +78,10 @@
content_got = b''
for chunk in result:
content_got += chunk

assert content_got == content


# 下载原文件到本地文件
result = bucket.get_object_to_file(key, filename)

Expand All @@ -62,12 +91,52 @@

os.remove(filename)

# 下载部分文件
result = bucket.get_object(key, byte_range=(0, 1024))

# 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS)，此模式下只提供对象整体上传下载操作
bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name,
crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, '1234'))
# 验证一下
content_got = b''
for chunk in result:
content_got += chunk
assert content_got == content[0:1025]

key1 = 'motto-copy.txt'
# 分片上传
part_a = b'a' * 1024 * 100
part_b = b'b' * 1024 * 100
part_c = b'c' * 1024 * 100
multi_content = [part_a, part_b, part_c]

parts = []
data_size = 100 * 1024 * 3
part_size = 100 * 1024
multi_key = "test_crypto_multipart"

context = models.MultipartUploadCryptoContext(data_size, part_size)
res = bucket.init_multipart_upload(multi_key, upload_context=context)
upload_id = res.upload_id

# 分片上传
for i in range(3):
result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], upload_context=context)
parts.append(oss2.models.PartInfo(i+1, result.etag, size=part_size, part_crc=result.crc))

# 完成上传
result = bucket.complete_multipart_upload(multi_key, upload_id, parts)

# 下载全部文件
result = bucket.get_object(multi_key)

# 验证一下
content_got = b''
for chunk in result:
content_got += chunk
assert content_got[0:102400] == part_a
assert content_got[102400:204800] == part_b
assert content_got[204800:307200] == part_c

# 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS)
bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name,
crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk))

# 上传文件
bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)})
Expand All @@ -93,4 +162,50 @@
with open(filename, 'rb') as fileobj:
assert fileobj.read() == content

os.remove(filename)
os.remove(filename)

# 下载部分文件
result = bucket.get_object(key, byte_range=(0, 1024))

# 验证一下
content_got = b''
for chunk in result:
content_got += chunk
assert content_got == content[0:1025]

"""
分片上传
"""
# 初始化上传分片
part_a = b'a' * 1024 * 100
part_b = b'b' * 1024 * 100
part_c = b'c' * 1024 * 100
multi_content = [part_a, part_b, part_c]

parts = []
data_size = 100 * 1024 * 3
part_size = 100 * 1024
multi_key = "test_crypto_multipart"

context = models.MultipartUploadCryptoContext(data_size, part_size)
res = bucket.init_multipart_upload(multi_key, upload_context=context)
upload_id = res.upload_id

# 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。
for i in range(3):
result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], upload_context=context)
parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc))

# 完成上传
result = bucket.complete_multipart_upload(multi_key, upload_id, parts)

# 下载全部文件
result = bucket.get_object(multi_key)

# 验证一下
content_got = b''
for chunk in result:
content_got += chunk
assert content_got[0:102400] == part_a
assert content_got[102400:204800] == part_b
assert content_got[204800:307200] == part_c

0 comments on commit 818652a

Please sign in to comment.