Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Log #130

Merged
merged 40 commits into from
Sep 17, 2018
Merged

Log #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
7097a61
add check_crc to get_object_to_file
yangwanyuan Aug 6, 2018
26b5edb
add crc64_combine.py to support crc64 combine feature
yangwanyuan Aug 7, 2018
8d991f5
add crc64_combine and check_crc to Multipart Upload
yangwanyuan Aug 8, 2018
2531807
add check_crc to resumable download
yangwanyuan Aug 9, 2018
6d64a89
fix test and unittest error compare to release 2.5.0
yangwanyuan Aug 10, 2018
73a9cd9
add crc related tests and examples
yangwanyuan Aug 13, 2018
33460b3
fix code to make it compatiable with python3
yangwanyuan Aug 13, 2018
b6b86eb
add detail logs
hangzws Aug 21, 2018
a3cb6f2
add thread id in all logs & add logs for init auth
hangzws Aug 21, 2018
0580c8e
add some check crc functional test
yangwanyuan Aug 21, 2018
c6f0829
add extranet_endpoint&intranet_endpoint&storage_class in list buckets…
hangzws Aug 22, 2018
7af0828
quote source_key in upload part copy
hangzws Aug 22, 2018
5af6f58
add support for ipv6
hangzws Aug 22, 2018
5c55302
add requestHeader
yangwanyuan Aug 22, 2018
3791c02
add requestHeader unittest
yangwanyuan Aug 22, 2018
7ac2935
add server crypto example
yangwanyuan Aug 22, 2018
6857d34
fix setServerSideEncryto bug
yangwanyuan Aug 23, 2018
fed6c08
replace some header string with variable in headers.py
yangwanyuan Aug 23, 2018
87e93ac
add process object & put/get object with signed url
hangzws Aug 23, 2018
8701e4e
add api for put/get with signed url and tests
hangzws Aug 28, 2018
afa347c
Merge branch 'log' into ywy-crc-dev
yangwanyuan Aug 31, 2018
4b97c18
Merge branch 'log' into ywy-cmkid-dev
yangwanyuan Aug 31, 2018
4f7a381
add test case for process object
hangzws Sep 2, 2018
98f7597
use bytes as base64 encode param
hangzws Sep 3, 2018
bf877f1
transfer encode base64 bytes to string
hangzws Sep 3, 2018
176bdc6
add comment for process object
hangzws Sep 4, 2018
9ac128d
fix code after review
yangwanyuan Sep 8, 2018
a4c7efe
fix code after review
yangwanyuan Sep 8, 2018
9ee714a
fix unittests in python 3
yangwanyuan Sep 8, 2018
5fd07b4
code review fix
yangwanyuan Sep 9, 2018
036a21b
add crc64 combine unittest
yangwanyuan Sep 9, 2018
f559c9f
remove note in crc64_combine, remove request_id from download
yangwanyuan Sep 14, 2018
60361f7
Merge pull request #125 from yangwanyuan/ywy-crc-dev
hangzws Sep 14, 2018
5578759
Merge branch 'log' into ywy-cmkid-dev
hangzws Sep 14, 2018
a776c11
Merge pull request #126 from yangwanyuan/ywy-cmkid-dev
hangzws Sep 14, 2018
33c2aa1
Merge branch 'log' of github.com:aliyun/aliyun-oss-python-sdk into log
hangzws Sep 15, 2018
8f331e0
variable&func naming specification modification
hangzws Sep 15, 2018
e1e8c9d
add ut for init
hangzws Sep 16, 2018
e7bafe5
fix process object return & move ut to ft
hangzws Sep 16, 2018
4dad6d5
convert http response to string while py3
hangzws Sep 16, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/object_callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
upload_id = bucket.init_multipart_upload(key).upload_id
# 上传分片
result = bucket.upload_part(key, upload_id, 1, content)
parts.append(oss2.models.PartInfo(1, result.etag))
parts.append(oss2.models.PartInfo(1, result.etag, size = len(content), part_crc = result.crc))
# 完成上传并回调
result = bucket.complete_multipart_upload(key, upload_id, parts, headers)

Expand Down
8 changes: 4 additions & 4 deletions examples/object_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def _prepare_temp_file(content):
encode_md5 = calculate_data_md5(content)
for i in range(3):
result = bucket.upload_part(key, upload_id, i+1, content, headers={'Content-MD5': encode_md5})
parts.append(oss2.models.PartInfo(i+1, result.etag))
parts.append(oss2.models.PartInfo(i+1, result.etag, size = len(content), part_crc = result.crc))

# 完成上传并回调
result = bucket.complete_multipart_upload(key, upload_id, parts)
Expand Down Expand Up @@ -156,7 +156,7 @@ def _prepare_temp_file(content):
# 上传分片,每个分片单独CRC校验
for i in range(3):
result = bucket.upload_part(key, upload_id, i+1, content)
parts.append(oss2.models.PartInfo(i+1, result.etag))
parts.append(oss2.models.PartInfo(i+1, result.etag, size = len(content), part_crc = result.crc))

# 完成上传并回调
result = bucket.complete_multipart_upload(key, upload_id, parts)
Expand All @@ -178,14 +178,14 @@ def _prepare_temp_file(content):
assert result.client_crc == result.server_crc


# 下载文件到本地
# 下载文件到本地,默认开启CRC校验
local_file = 'download.txt'
result = bucket.get_object_to_file(key, local_file)
os.remove(local_file)
assert result.client_crc == result.server_crc


# 断点续传下载,不支持CRC校验,可以用如下方法校验
# 断点续传下载, 自动开启CRC校验,也可以用如下方法校验
oss2.resumable_download(bucket, key, local_file,
multiget_threshold=200*1024,
part_size=100*1024,
Expand Down
2 changes: 1 addition & 1 deletion examples/object_progress.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def _prepare_temp_file(content):
# 上传分片
for i in range(3):
result = bucket.upload_part(key, upload_id, i+1, content, progress_callback=percentage)
parts.append(oss2.models.PartInfo(i+1, result.etag))
parts.append(oss2.models.PartInfo(i+1, result.etag, size = len(content), part_crc = result.crc))

# 完成上传并回调
result = bucket.complete_multipart_upload(key, upload_id, parts)
Expand Down
60 changes: 60 additions & 0 deletions examples/object_server_crypto.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-

import os
import shutil

import oss2
from oss2.headers import RequestHeader


# 以下代码展示了其用服务端加密功能的各项操作


# 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。
# 通过环境变量获取,或者把诸如“<你的AccessKeyId>”替换成真实的AccessKeyId等。
#
# 以杭州区域为例,Endpoint可以是:
# http://oss-cn-hangzhou.aliyuncs.com
# https://oss-cn-hangzhou.aliyuncs.com
# 分别以HTTP、HTTPS协议访问。
access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '<你的AccessKeyId>')
access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '<你的AccessKeySecret>')
bucket_name = os.getenv('OSS_TEST_BUCKET', '<你的Bucket>')
endpoint = os.getenv('OSS_TEST_ENDPOINT', '<你的访问域名>')


# 确认上面的参数都填写正确了
for param in (access_key_id, access_key_secret, bucket_name, endpoint):
assert '<' not in param, '请设置参数:' + param


key = 'server-crypto.txt'
content = b'a' * 1024 * 1024

# 创建Bucket对象,所有Object相关的接口都可以通过Bucket对象来进行
bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name)


# 上传文件使用服务端AES256进行加密
myHeader = RequestHeader()
myHeader.set_server_side_encryption("AES256")
bucket.put_object(key, content, headers = myHeader)

# 下载文件验证一下
result = bucket.get_object(key)
content_got = b''
for chunk in result:
content_got += chunk
assert content_got == content

# 上传文件使用服务端KMS进行加密
myHeader = RequestHeader()
myHeader.set_server_side_encryption("KMS", cmk_id = "11111")
bucket.put_object(key, content, headers = myHeader)

# 下载文件验证一下
result = bucket.get_object(key)
content_got = b''
for chunk in result:
content_got += chunk
assert content_got == content
12 changes: 6 additions & 6 deletions examples/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,18 @@ def random_string(n):
upload_id = bucket.init_multipart_upload(key).upload_id

# 逐个上传分片
# 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于num_to_upload
# 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于size_to_upload
with open(filename, 'rb') as fileobj:
parts = []
part_number = 1
offset = 0
while offset < total_size:
num_to_upload = min(part_size, total_size - offset)
size_to_upload = min(part_size, total_size - offset)
result = bucket.upload_part(key, upload_id, part_number,
oss2.SizedFileAdapter(fileobj, num_to_upload))
parts.append(oss2.models.PartInfo(part_number, result.etag))
oss2.SizedFileAdapter(fileobj, size_to_upload))
parts.append(oss2.models.PartInfo(part_number, result.etag, size = size_to_upload, part_crc = result.crc))

offset += num_to_upload
offset += size_to_upload
part_number += 1

# 完成分片上传
Expand All @@ -91,4 +91,4 @@ def random_string(n):
assert bucket.get_object(key).read() == fileobj.read()


os.remove(filename)
os.remove(filename)
32 changes: 32 additions & 0 deletions oss2/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,35 @@
from .models import BUCKET_STORAGE_CLASS_STANDARD, BUCKET_STORAGE_CLASS_IA, BUCKET_STORAGE_CLASS_ARCHIVE

from .crypto import LocalRsaProvider, AliKMSProvider
import logging

logger = logging.getLogger('oss2')


def set_file_logger(file_path, name="oss2", level=logging.INFO, format_string=None):
global logger
if not format_string:
format_string = "%(asctime)s %(name)s [%(levelname)s] %(thread)d : %(message)s"
logger = logging.getLogger(name)
logger.setLevel(level)
fh = logging.FileHandler(file_path)
fh.setLevel(level)
formatter = logging.Formatter(format_string)
fh.setFormatter(formatter)
logger.addHandler(fh)


def set_stream_logger(name='oss2', level=logging.DEBUG, format_string=None):
global logger
if not format_string:
format_string = "%(asctime)s %(name)s [%(levelname)s] %(thread)d : %(message)s"
logger = logging.getLogger(name)
logger.setLevel(level)
fh = logging.StreamHandler()
fh.setLevel(level)
formatter = logging.Formatter(format_string)
fh.setFormatter(formatter)
logger.addHandler(fh)


set_stream_logger('oss2', logging.INFO)