Merge pull request #42 from aliyun/fix-object-exists

fix object exist

oss2/api.py

@@ -503,25 +503,40 @@ def head_object(self, key, headers=None):
         """
         resp = self.__do_object('HEAD', key, headers=headers)
         return HeadObjectResult(resp)
+
+    def get_object_meta(self, key):
+        """获取文件基本元信息，包括该Object的ETag、Size（文件大小）、LastModified，并不返回其内容。
+
+        HTTP响应的头部包含了文件基本元信息，可以通过 `GetObjectMetaResult` 的 `last_modified`，`content_length`,`etag` 成员获得。
+
+        :param key: 文件名
+
+        :return: :class:`GetObjectMetaResult <oss2.models.GetObjectMetaResult>`
+
+        :raises: 如果文件不存在，则抛出 :class:`NoSuchKey <oss2.exceptions.NoSuchKey>` ；还可能抛出其他异常
+        """
+        resp = self.__do_object('GET', key, params={'objectMeta': ''})
+        return GetObjectMetaResult(resp)
 
     def object_exists(self, key):
         """如果文件存在就返回True，否则返回False。如果Bucket不存在，或是发生其他错误，则抛出异常。"""
 
         # 如果我们用head_object来实现的话，由于HTTP HEAD请求没有响应体，只有响应头部，这样当发生404时，
         # 我们无法区分是NoSuchBucket还是NoSuchKey错误。
         #
-        # 下面的实现是通过if-modified-since头部，把date设为当前时间24小时后，这样如果文件存在，则会返回
-        # 304 (NotModified)；不存在，则会返回NoSuchKey
-        date = oss2.utils.http_date(int(time.time()) + 24 * 60 * 60)
+        # 2.2.0之前的实现是通过get_object的if-modified-since头部，把date设为当前时间24小时后，这样如果文件存在，则会返回
+        # 304 (NotModified)；不存在，则会返回NoSuchKey。get_object会受回源的影响，如果配置会404回源，get_object会判断错误。
+        # 
+        # 目前的实现是通过get_object_meta判断文件是否存在。
 
         try:
-            self.get_object(key, headers={'if-modified-since': date})
-        except exceptions.NotModified:
-            return True
+            self.get_object_meta(key)
         except exceptions.NoSuchKey:
             return False
-        else:
-            raise exceptions.ClientError('Client time varies too much from server?')  # pragma: no cover
+        except:
+            raise
+
+        return True
 
     def copy_object(self, source_bucket_name, source_key, target_key, headers=None):
         """拷贝一个文件到当前Bucket。

oss2/auth.py

@@ -16,7 +16,7 @@ class Auth(object):
         ['response-content-type', 'response-content-language',
          'response-cache-control', 'logging', 'response-content-encoding',
          'acl', 'uploadId', 'uploads', 'partNumber', 'group', 'link',
-         'delete', 'website', 'location', 'objectInfo',
+         'delete', 'website', 'location', 'objectInfo', 'objectMeta',
          'response-expires', 'response-content-disposition', 'cors', 'lifecycle',
          'restore', 'qos', 'referer', 'append', 'position', 'security-token',
          'live', 'comp', 'status', 'vod', 'startTime', 'endTime', 'x-oss-process']

oss2/models.py

@@ -76,6 +76,20 @@ def __init__(self, resp):
         self.etag = _get_etag(self.headers)
 
 
+class GetObjectMetaResult(RequestResult):
+    def __init__(self, resp):
+        super(GetObjectMetaResult, self).__init__(resp)
+
+        #: 文件最后修改时间，类型为int。参考 :ref:`unix_time` 。
+        self.last_modified = _hget(self.headers, 'last-modified', http_to_unixtime)
+
+        #: Content-Length，文件大小，类型为int。
+        self.content_length = _hget(self.headers, 'content-length', int)
+
+        #: HTTP ETag
+        self.etag = _get_etag(self.headers)
+
+
 class GetObjectResult(HeadObjectResult):
     def __init__(self, resp, progress_callback=None, crc_enabled=False):
         super(GetObjectResult, self).__init__(resp)

tests/test_mock_object.py

@@ -128,26 +128,23 @@ def test_head(self, do_request):
 
     @patch('oss2.Session.do_request')
     def test_object_exists_true(self, do_request):
-        request_text = '''GET /sbowspxjhmccpmesjqcwagfw HTTP/1.1
+        request_text = '''GET /sbowspxjhmccpmesjqcwagfw?objectMeta HTTP/1.1
 Host: ming-oss-share.oss-cn-hangzhou.aliyuncs.com
 Accept-Encoding: identity
 Connection: keep-alive
-if-modified-since: Sun, 13 Dec 2015 00:37:17 GMT
 date: Sat, 12 Dec 2015 00:37:17 GMT
 User-Agent: aliyun-sdk-python/2.0.2(Windows/7/;3.3.3)
 Accept: */*
 authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''
 
-        response_text = '''HTTP/1.1 304 Not Modified
-Server: AliyunOSS
-Date: Sat, 12 Dec 2015 00:37:17 GMT
-Content-Type: application/octet-stream
-Connection: keep-alive
+        response_text = '''HTTP/1.1 200 OK
 x-oss-request-id: 566B6C3D010B7A4314D2253D
-Accept-Ranges: bytes
-ETag: "5EB63BBBE01EEED093CB22BB8F5ACDC3"
+Date: Sat, 12 Dec 2015 00:37:17 GMT
+ETag: "5B3C1A2E053D763E1B002CC607C5A0FE"
 Last-Modified: Sat, 12 Dec 2015 00:37:17 GMT
-x-oss-object-type: Normal'''
+Content-Length: 344606
+Connection: keep-alive
+Server: AliyunOSS'''
 
         req_info = unittests.common.mock_response(do_request, response_text)
 
@@ -156,15 +153,14 @@ def test_object_exists_true(self, do_request):
 
     @patch('oss2.Session.do_request')
     def test_object_exists_false(self, do_request):
-        request_text = '''GET /sbowspxjhmccpmesjqcwagfw HTTP/1.1
+        request_text = '''GET /sbowspxjhmccpmesjqcwagfw?objectMeta HTTP/1.1
 Host: ming-oss-share.oss-cn-hangzhou.aliyuncs.com
 Accept-Encoding: identity
 Connection: keep-alive
-if-modified-since: Sun, 13 Dec 2015 00:37:17 GMT
 date: Sat, 12 Dec 2015 00:37:17 GMT
 User-Agent: aliyun-sdk-python/2.0.2(Windows/7/;3.3.3)
 Accept: */*
-authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''
+authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''     
 
         response_text = '''HTTP/1.1 404 Not Found
 Server: AliyunOSS
@@ -186,6 +182,67 @@ def test_object_exists_false(self, do_request):
         req_info = unittests.common.mock_response(do_request, response_text)
         self.assertTrue(not unittests.common.bucket().object_exists('sbowspxjhmccpmesjqcwagfw'))
         self.assertRequest(req_info, request_text)
+
+    @patch('oss2.Session.do_request')
+    def test_object_exists_exception(self, do_request):
+        request_text = '''GET /sbowspxjhmccpmesjqcwagfw?objectMeta HTTP/1.1
+Host: ming-oss-share.oss-cn-hangzhou.aliyuncs.com
+Accept-Encoding: identity
+Connection: keep-alive
+date: Sat, 12 Dec 2015 00:37:17 GMT
+User-Agent: aliyun-sdk-python/2.0.2(Windows/7/;3.3.3)
+Accept: */*
+authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''     
+
+        response_text = '''HTTP/1.1 404 Not Found
+Server: AliyunOSS
+Date: Sat, 12 Dec 2015 00:37:17 GMT
+Content-Type: application/xml
+Content-Length: 287
+Connection: keep-alive
+x-oss-request-id: 566B6C3D6086505A0CFF0F68
+
+<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+  <Code>NoSuchBucket</Code>
+  <Message>The specified bucket does not exist.</Message>
+  <RequestId>566B6C3D6086505A0CFF0F68</RequestId>
+  <HostId>ming-oss-share.oss-cn-hangzhou.aliyuncs.com</HostId>
+  <Bucket>ming-oss-share</Bucket>
+</Error>'''
+
+        unittests.common.mock_response(do_request, response_text)
+        self.assertRaises(oss2.exceptions.NoSuchBucket, unittests.common.bucket().object_exists, 'sbowspxjhmccpmesjqcwagfw')
+
+    @patch('oss2.Session.do_request')
+    def test_get_object_meta(self, do_request):
+        request_text = '''GET /sbowspxjhmccpmesjqcwagfw?objectMeta HTTP/1.1
+Host: ming-oss-share.oss-cn-hangzhou.aliyuncs.com
+Accept-Encoding: identity
+Connection: keep-alive
+date: Sat, 12 Dec 2015 00:37:17 GMT
+User-Agent: aliyun-sdk-python/2.0.2(Windows/7/;3.3.3)
+Accept: */*
+authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''
+
+        response_text = '''HTTP/1.1 200 OK
+x-oss-request-id: 566B6C3D010B7A4314D2253D
+Date: Sat, 12 Dec 2015 00:37:17 GMT
+ETag: "5B3C1A2E053D763E1B002CC607C5A0FE"
+Last-Modified: Sat, 12 Dec 2015 00:37:17 GMT
+Content-Length: 344606
+Connection: keep-alive
+Server: AliyunOSS'''
+
+        req_info = unittests.common.mock_response(do_request, response_text)
+
+        result = unittests.common.bucket().get_object_meta('sbowspxjhmccpmesjqcwagfw')
+
+        self.assertRequest(req_info, request_text)
+
+        self.assertEqual(result.last_modified, 1449880637)
+        self.assertEqual(result.content_length, 344606)
+        self.assertEqual(result.etag, '5B3C1A2E053D763E1B002CC607C5A0FE')
 
     @patch('oss2.Session.do_request')
     def test_get(self, do_request):

tests/test_object.py

@@ -4,7 +4,7 @@
 import filecmp
 import calendar
 
-from oss2.exceptions import (ClientError, RequestError,
+from oss2.exceptions import (ClientError, RequestError, NoSuchBucket,
                              NotFound, NoSuchKey, Conflict, PositionNotEqualToLength, ObjectNotAppendable)
 from common import *
 
@@ -322,6 +322,10 @@ def test_object_acl(self):
 
     def test_object_exists(self):
         key = self.random_key()
+
+        auth = oss2.Auth(OSS_ID, OSS_SECRET)
+        bucket = oss2.Bucket(auth, OSS_ENDPOINT, random_string(63).lower())
+        self.assertRaises(NoSuchBucket, bucket.object_exists, key)
 
         self.assertTrue(not self.bucket.object_exists(key))
 
@@ -337,6 +341,28 @@ def test_user_meta(self):
         headers = self.bucket.get_object(key).headers
         self.assertEqual(headers['x-oss-meta-key1'], 'value1')
         self.assertEqual(headers['x-oss-meta-key2'], 'value2')
+
+    def test_get_object_meta(self):
+        key = self.random_key()
+        content = 'hello'
+
+        # bucket no exist
+        auth = oss2.Auth(OSS_ID, OSS_SECRET)
+        bucket = oss2.Bucket(auth, OSS_ENDPOINT, random_string(63).lower())
+
+        self.assertRaises(NoSuchBucket, bucket.get_object_meta, key)
+
+        # object no exist
+        self.assertRaises(NoSuchKey, self.bucket.get_object_meta, key)
+
+        self.bucket.put_object(key, content)
+
+        # get meta normal
+        result = self.bucket.get_object_meta(key)
+
+        self.assertTrue(result.last_modified > 1472128796)
+        self.assertEqual(result.content_length, len(content))
+        self.assertEqual(result.etag, '5D41402ABC4B2A76B9719D911017C592')
 
     def test_progress(self):
         stats = {'previous': -1}

unittests/test_object.py

@@ -129,26 +129,23 @@ def test_head(self, do_request):
 
     @patch('oss2.Session.do_request')
     def test_object_exists_true(self, do_request):
-        request_text = '''GET /sbowspxjhmccpmesjqcwagfw HTTP/1.1
+        request_text = '''GET /sbowspxjhmccpmesjqcwagfw?objectMeta HTTP/1.1
 Host: ming-oss-share.oss-cn-hangzhou.aliyuncs.com
 Accept-Encoding: identity
 Connection: keep-alive
-if-modified-since: Sun, 13 Dec 2015 00:37:17 GMT
 date: Sat, 12 Dec 2015 00:37:17 GMT
 User-Agent: aliyun-sdk-python/2.0.2(Windows/7/;3.3.3)
 Accept: */*
 authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''
 
-        response_text = '''HTTP/1.1 304 Not Modified
-Server: AliyunOSS
-Date: Sat, 12 Dec 2015 00:37:17 GMT
-Content-Type: application/octet-stream
-Connection: keep-alive
+        response_text = '''HTTP/1.1 200 OK
 x-oss-request-id: 566B6C3D010B7A4314D2253D
-Accept-Ranges: bytes
-ETag: "5EB63BBBE01EEED093CB22BB8F5ACDC3"
+Date: Sat, 12 Dec 2015 00:37:17 GMT
+ETag: "5B3C1A2E053D763E1B002CC607C5A0FE"
 Last-Modified: Sat, 12 Dec 2015 00:37:17 GMT
-x-oss-object-type: Normal'''
+Content-Length: 344606
+Connection: keep-alive
+Server: AliyunOSS'''
 
         req_info = mock_response(do_request, response_text)
 
@@ -157,15 +154,14 @@ def test_object_exists_true(self, do_request):
 
     @patch('oss2.Session.do_request')
     def test_object_exists_false(self, do_request):
-        request_text = '''GET /sbowspxjhmccpmesjqcwagfw HTTP/1.1
+        request_text = '''GET /sbowspxjhmccpmesjqcwagfw?objectMeta HTTP/1.1
 Host: ming-oss-share.oss-cn-hangzhou.aliyuncs.com
 Accept-Encoding: identity
 Connection: keep-alive
-if-modified-since: Sun, 13 Dec 2015 00:37:17 GMT
 date: Sat, 12 Dec 2015 00:37:17 GMT
 User-Agent: aliyun-sdk-python/2.0.2(Windows/7/;3.3.3)
 Accept: */*
-authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''
+authorization: OSS ZCDmm7TPZKHtx77j:wopWcmMd/70eNKYOc9M6ZA21yY8='''     
 
         response_text = '''HTTP/1.1 404 Not Found
 Server: AliyunOSS