From 9f35b6252d60600fccfc4951efa57732598efc6c Mon Sep 17 00:00:00 2001 From: Louis Garman <75728+leg100@users.noreply.github.com> Date: Thu, 15 Aug 2019 12:29:56 +0100 Subject: [PATCH 1/3] Typo. --- gcsfs/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcsfs/core.py b/gcsfs/core.py index 9e157e28..699753fd 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -257,7 +257,7 @@ class GCSFileSystem(fsspec.AbstractFileSystem): across an insecure network. check_connection: bool When token=None, gcsfs will attempt various methods of establishing - credentials, falling back to anon. It is possible for a methoc to + credentials, falling back to anon. It is possible for a method to find credentials in the system that turn out not to be valid. Setting this parameter to True will ensure that an actual operation is attempted before deciding that credentials are valid. From 4b90719d968559bd576df330337639842cf46fd7 Mon Sep 17 00:00:00 2001 From: Louis Garman <75728+leg100@users.noreply.github.com> Date: Thu, 15 Aug 2019 12:30:10 +0100 Subject: [PATCH 2/3] Checking public bucket violates VPC service ctrls --- gcsfs/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcsfs/core.py b/gcsfs/core.py index 699753fd..2c196d3d 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -273,7 +273,7 @@ class GCSFileSystem(fsspec.AbstractFileSystem): def __init__(self, project=DEFAULT_PROJECT, access='full_control', token=None, block_size=None, consistency='none', cache_timeout=None, secure_serialize=True, - check_connection=True, requests_timeout=None, **kwargs): + check_connection=False, requests_timeout=None, **kwargs): super().__init__(self, **kwargs) pars = (project, access, token, block_size, consistency, cache_timeout) if access not in self.scopes: From a0acf55fe9628025c81de8cda7a2e71cbb525df5 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 16 Aug 2019 12:08:19 -0400 Subject: [PATCH 3/3] update recording --- gcsfs/tests/recordings/test_rm_recursive.yaml | 271 +++++++++--------- gcsfs/tests/test_core.py | 2 +- gcsfs/tests/test_fuse.py | 3 - gcsfs/tests/test_mapping.py | 2 +- 4 files changed, 144 insertions(+), 134 deletions(-) diff --git a/gcsfs/tests/recordings/test_rm_recursive.yaml b/gcsfs/tests/recordings/test_rm_recursive.yaml index 0eeffafd..f6ffa45d 100644 --- a/gcsfs/tests/recordings/test_rm_recursive.yaml +++ b/gcsfs/tests/recordings/test_rm_recursive.yaml @@ -17,7 +17,7 @@ interactions: response: body: string: !!binary | - H4sIALODF10C/4WOQQoCMQxFr1KylnZAcDFLL1JKjZ1inZQkM62Id9fqAdw+3uf9J4QYUcQr3XCF + H4sIAOjUVl0C/4WOQQoCMQxFr1KylnZAcDFLL1JKjZ1inZQkM62Id9fqAdw+3uf9J4QYUcQr3XCF 2UDvHQ4GsNfMKD4PeDxN04dJpIrDWVSrzM611mwiSgVDzWIj3V3YdHEX3EWJQ0J73UrxkVZlKubv jvOOo/594/Xxy50xMDK83tzjrvutAAAA headers: @@ -54,7 +54,7 @@ interactions: Connection: - keep-alive Content-Length: - - '25' + - '24' Content-Type: - application/json method: POST @@ -93,17 +93,19 @@ interactions: uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa response: body: - string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n - \ \"reason\": \"notFound\",\n \"message\": \"No such object: gcsfs-testing/tmp/test/a\"\n - \ }\n ],\n \"code\": 404,\n \"message\": \"No such object: gcsfs-testing/tmp/test/a\"\n - }\n}\n" + string: "{\n \"error\": {\n \"code\": 404,\n \"message\": \"No such object: + gcsfs-testing/tmp/test/a\",\n \"errors\": [\n {\n \"message\": + \"No such object: gcsfs-testing/tmp/test/a\",\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\"\n }\n ]\n }\n}\n" headers: Cache-Control: - - private, max-age=0 + - no-cache, no-store, max-age=0, must-revalidate Content-Length: - - '227' + - '253' Content-Type: - application/json; charset=UTF-8 + Pragma: + - no-cache Server: - UploadServer Vary: @@ -135,7 +137,7 @@ interactions: Cache-Control: - private, max-age=0 Content-Length: - - '227' + - '225' Content-Type: - application/json; charset=UTF-8 Server: @@ -169,7 +171,7 @@ interactions: Cache-Control: - private, max-age=0 Content-Length: - - '227' + - '225' Content-Type: - application/json; charset=UTF-8 Server: @@ -195,17 +197,19 @@ interactions: uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fd response: body: - string: "{\n \"error\": {\n \"errors\": [\n {\n \"domain\": \"global\",\n - \ \"reason\": \"notFound\",\n \"message\": \"No such object: gcsfs-testing/tmp/test/d\"\n - \ }\n ],\n \"code\": 404,\n \"message\": \"No such object: gcsfs-testing/tmp/test/d\"\n - }\n}\n" + string: "{\n \"error\": {\n \"code\": 404,\n \"message\": \"No such object: + gcsfs-testing/tmp/test/d\",\n \"errors\": [\n {\n \"message\": + \"No such object: gcsfs-testing/tmp/test/d\",\n \"domain\": \"global\",\n + \ \"reason\": \"notFound\"\n }\n ]\n }\n}\n" headers: Cache-Control: - - private, max-age=0 + - no-cache, no-store, max-age=0, must-revalidate Content-Length: - - '227' + - '253' Content-Type: - application/json; charset=UTF-8 + Pragma: + - no-cache Server: - UploadServer Vary: @@ -238,9 +242,9 @@ interactions: Content-Length: - '0' Content-Type: - - text/html; charset=UTF-8 + - text/plain; charset=utf-8 Location: - - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2UrwjqgFe4zCpfy4cR5Gf53WW00MBH_HQ8Yd8qBks3XNiNsmOEfqzjE_KvgIc-389yPK-2_zA7gtKY-OT9gUe89SeTzQqg + - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2Uqd7ImswhSWJFZ2LlXoX5UUJcmANAmmObN7Ipioj1ouz8cogQUNFZpAANY2MfUR0G8x2lfw_Kdm-zf33eo0xdx1J4e6HQ Pragma: - no-cache Server: @@ -267,26 +271,26 @@ interactions: Content-Type: - application/octet-stream method: POST - uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2UrwjqgFe4zCpfy4cR5Gf53WW00MBH_HQ8Yd8qBks3XNiNsmOEfqzjE_KvgIc-389yPK-2_zA7gtKY-OT9gUe89SeTzQqg + uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2Uqd7ImswhSWJFZ2LlXoX5UUJcmANAmmObN7Ipioj1ouz8cogQUNFZpAANY2MfUR0G8x2lfw_Kdm-zf33eo0xdx1J4e6HQ response: body: - string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/1561822132714573\",\n + string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/1565971690971479\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a\",\n - \"name\": \"a\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1561822132714573\",\n - \"metageneration\": \"1\",\n \"timeCreated\": \"2019-06-29T15:28:52.714Z\",\n - \"updated\": \"2019-06-29T15:28:52.714Z\",\n \"storageClass\": \"STANDARD\",\n - \"timeStorageClassUpdated\": \"2019-06-29T15:28:52.714Z\",\n \"size\": \"0\",\n - \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a?generation=1561822132714573&alt=media\",\n - \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CM34sOuAj+MCEAE=\"\n}\n" + \"name\": \"a\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1565971690971479\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2019-08-16T16:08:10.971Z\",\n + \"updated\": \"2019-08-16T16:08:10.971Z\",\n \"storageClass\": \"MULTI_REGIONAL\",\n + \"timeStorageClassUpdated\": \"2019-08-16T16:08:10.971Z\",\n \"size\": \"0\",\n + \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a?generation=1565971690971479&alt=media\",\n + \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CNfC+ZLjh+QCEAE=\"\n}\n" headers: Cache-Control: - no-cache, no-store, max-age=0, must-revalidate Content-Length: - - '645' + - '647' Content-Type: - application/json; charset=UTF-8 ETag: - - CM34sOuAj+MCEAE= + - CNfC+ZLjh+QCEAE= Pragma: - no-cache Server: @@ -321,9 +325,9 @@ interactions: Content-Length: - '0' Content-Type: - - text/html; charset=UTF-8 + - text/plain; charset=utf-8 Location: - - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2Uqq15VMP7ZMGOpTS_ae1NqfisTUNaBxQ-xiVRkwIowszXTiJIrJUl7AccQCtAZXEY2ZHTCJijmZ_ehCwCu71zD-vGiBSA + - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2Ur1iH6BFUc5a1FTTt8k-FHVrrlMIUk3juy53T-vCC4zS59U9iazIbrvf8tmYLiN9TUPtIzjebX7e5AojKgyz1TRq3VyBg Pragma: - no-cache Server: @@ -350,26 +354,26 @@ interactions: Content-Type: - application/octet-stream method: POST - uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2Uqq15VMP7ZMGOpTS_ae1NqfisTUNaBxQ-xiVRkwIowszXTiJIrJUl7AccQCtAZXEY2ZHTCJijmZ_ehCwCu71zD-vGiBSA + uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2Ur1iH6BFUc5a1FTTt8k-FHVrrlMIUk3juy53T-vCC4zS59U9iazIbrvf8tmYLiN9TUPtIzjebX7e5AojKgyz1TRq3VyBg response: body: - string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/b/1561822133331212\",\n + string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/b/1565971691667641\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a%2Fb\",\n - \"name\": \"a/b\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1561822133331212\",\n - \"metageneration\": \"1\",\n \"timeCreated\": \"2019-06-29T15:28:53.331Z\",\n - \"updated\": \"2019-06-29T15:28:53.331Z\",\n \"storageClass\": \"STANDARD\",\n - \"timeStorageClassUpdated\": \"2019-06-29T15:28:53.331Z\",\n \"size\": \"0\",\n - \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fb?generation=1561822133331212&alt=media\",\n - \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CIzK1uuAj+MCEAE=\"\n}\n" + \"name\": \"a/b\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1565971691667641\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2019-08-16T16:08:11.667Z\",\n + \"updated\": \"2019-08-16T16:08:11.667Z\",\n \"storageClass\": \"MULTI_REGIONAL\",\n + \"timeStorageClassUpdated\": \"2019-08-16T16:08:11.667Z\",\n \"size\": \"0\",\n + \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fb?generation=1565971691667641&alt=media\",\n + \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CLmBpJPjh+QCEAE=\"\n}\n" headers: Cache-Control: - no-cache, no-store, max-age=0, must-revalidate Content-Length: - - '657' + - '659' Content-Type: - application/json; charset=UTF-8 ETag: - - CIzK1uuAj+MCEAE= + - CLmBpJPjh+QCEAE= Pragma: - no-cache Server: @@ -404,9 +408,9 @@ interactions: Content-Length: - '0' Content-Type: - - text/html; charset=UTF-8 + - text/plain; charset=utf-8 Location: - - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2Uo79iBYkqgFnOKSO4WEzDFyjTOtai7DAwQL7Wm0cQ_EUjDfy-NeOafpzXNCh_laW3bAetr5vkWdDJut-LBL4iIA5BTimg + - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2UqHWu1zZTJ5DI6TyQw6g-Ek88h0n9MIJqMhdFCMUoqvBBHwTROOr5lmnH162TLuUgGaOJgjdZHtRdbiXv1eaWDt2zA6qg Pragma: - no-cache Server: @@ -433,26 +437,26 @@ interactions: Content-Type: - application/octet-stream method: POST - uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2Uo79iBYkqgFnOKSO4WEzDFyjTOtai7DAwQL7Wm0cQ_EUjDfy-NeOafpzXNCh_laW3bAetr5vkWdDJut-LBL4iIA5BTimg + uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2UqHWu1zZTJ5DI6TyQw6g-Ek88h0n9MIJqMhdFCMUoqvBBHwTROOr5lmnH162TLuUgGaOJgjdZHtRdbiXv1eaWDt2zA6qg response: body: - string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/c/1561822134256605\",\n + string: "{\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/c/1565971692644338\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a%2Fc\",\n - \"name\": \"a/c\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1561822134256605\",\n - \"metageneration\": \"1\",\n \"timeCreated\": \"2019-06-29T15:28:54.256Z\",\n - \"updated\": \"2019-06-29T15:28:54.256Z\",\n \"storageClass\": \"STANDARD\",\n - \"timeStorageClassUpdated\": \"2019-06-29T15:28:54.256Z\",\n \"size\": \"0\",\n - \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fc?generation=1561822134256605&alt=media\",\n - \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CN2Hj+yAj+MCEAE=\"\n}\n" + \"name\": \"a/c\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": \"1565971692644338\",\n + \"metageneration\": \"1\",\n \"timeCreated\": \"2019-08-16T16:08:12.644Z\",\n + \"updated\": \"2019-08-16T16:08:12.644Z\",\n \"storageClass\": \"MULTI_REGIONAL\",\n + \"timeStorageClassUpdated\": \"2019-08-16T16:08:12.644Z\",\n \"size\": \"0\",\n + \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fc?generation=1565971692644338&alt=media\",\n + \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CPLP35Pjh+QCEAE=\"\n}\n" headers: Cache-Control: - no-cache, no-store, max-age=0, must-revalidate Content-Length: - - '657' + - '659' Content-Type: - application/json; charset=UTF-8 ETag: - - CN2Hj+yAj+MCEAE= + - CPLP35Pjh+QCEAE= Pragma: - no-cache Server: @@ -477,20 +481,20 @@ interactions: response: body: string: "{\n \"kind\": \"storage#objects\",\n \"prefixes\": [\n \"a/\"\n ],\n - \"items\": [\n {\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/1561822132714573\",\n + \"items\": [\n {\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/1565971690971479\",\n \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a\",\n \ \"name\": \"a\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": - \"1561822132714573\",\n \"metageneration\": \"1\",\n \"timeCreated\": - \"2019-06-29T15:28:52.714Z\",\n \"updated\": \"2019-06-29T15:28:52.714Z\",\n - \ \"storageClass\": \"STANDARD\",\n \"timeStorageClassUpdated\": \"2019-06-29T15:28:52.714Z\",\n - \ \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": - \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a?generation=1561822132714573&alt=media\",\n - \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CM34sOuAj+MCEAE=\"\n }\n ]\n}\n" + \"1565971690971479\",\n \"metageneration\": \"1\",\n \"timeCreated\": + \"2019-08-16T16:08:10.971Z\",\n \"updated\": \"2019-08-16T16:08:10.971Z\",\n + \ \"storageClass\": \"MULTI_REGIONAL\",\n \"timeStorageClassUpdated\": + \"2019-08-16T16:08:10.971Z\",\n \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n + \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a?generation=1565971690971479&alt=media\",\n + \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CNfC+ZLjh+QCEAE=\"\n }\n ]\n}\n" headers: Cache-Control: - private, max-age=0, must-revalidate, no-transform Content-Length: - - '754' + - '756' Content-Type: - application/json; charset=UTF-8 Server: @@ -515,29 +519,29 @@ interactions: response: body: string: "{\n \"kind\": \"storage#objects\",\n \"items\": [\n {\n \"kind\": - \"storage#object\",\n \"id\": \"gcsfs-testing/a/b/1561822133331212\",\n + \"storage#object\",\n \"id\": \"gcsfs-testing/a/b/1565971691667641\",\n \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a%2Fb\",\n \ \"name\": \"a/b\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": - \"1561822133331212\",\n \"metageneration\": \"1\",\n \"timeCreated\": - \"2019-06-29T15:28:53.331Z\",\n \"updated\": \"2019-06-29T15:28:53.331Z\",\n - \ \"storageClass\": \"STANDARD\",\n \"timeStorageClassUpdated\": \"2019-06-29T15:28:53.331Z\",\n - \ \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": - \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fb?generation=1561822133331212&alt=media\",\n - \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CIzK1uuAj+MCEAE=\"\n },\n {\n - \ \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/c/1561822134256605\",\n + \"1565971691667641\",\n \"metageneration\": \"1\",\n \"timeCreated\": + \"2019-08-16T16:08:11.667Z\",\n \"updated\": \"2019-08-16T16:08:11.667Z\",\n + \ \"storageClass\": \"MULTI_REGIONAL\",\n \"timeStorageClassUpdated\": + \"2019-08-16T16:08:11.667Z\",\n \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n + \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fb?generation=1565971691667641&alt=media\",\n + \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CLmBpJPjh+QCEAE=\"\n },\n {\n + \ \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/c/1565971692644338\",\n \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a%2Fc\",\n \ \"name\": \"a/c\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": - \"1561822134256605\",\n \"metageneration\": \"1\",\n \"timeCreated\": - \"2019-06-29T15:28:54.256Z\",\n \"updated\": \"2019-06-29T15:28:54.256Z\",\n - \ \"storageClass\": \"STANDARD\",\n \"timeStorageClassUpdated\": \"2019-06-29T15:28:54.256Z\",\n - \ \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": - \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fc?generation=1561822134256605&alt=media\",\n - \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CN2Hj+yAj+MCEAE=\"\n }\n ]\n}\n" + \"1565971692644338\",\n \"metageneration\": \"1\",\n \"timeCreated\": + \"2019-08-16T16:08:12.644Z\",\n \"updated\": \"2019-08-16T16:08:12.644Z\",\n + \ \"storageClass\": \"MULTI_REGIONAL\",\n \"timeStorageClassUpdated\": + \"2019-08-16T16:08:12.644Z\",\n \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n + \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a%2Fc?generation=1565971692644338&alt=media\",\n + \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CPLP35Pjh+QCEAE=\"\n }\n ]\n}\n" headers: Cache-Control: - private, max-age=0, must-revalidate, no-transform Content-Length: - - '1434' + - '1438' Content-Type: - application/json; charset=UTF-8 Server: @@ -562,20 +566,20 @@ interactions: response: body: string: "{\n \"kind\": \"storage#objects\",\n \"prefixes\": [\n \"a/\"\n ],\n - \"items\": [\n {\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/1561822132714573\",\n + \"items\": [\n {\n \"kind\": \"storage#object\",\n \"id\": \"gcsfs-testing/a/1565971690971479\",\n \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/a\",\n \ \"name\": \"a\",\n \"bucket\": \"gcsfs-testing\",\n \"generation\": - \"1561822132714573\",\n \"metageneration\": \"1\",\n \"timeCreated\": - \"2019-06-29T15:28:52.714Z\",\n \"updated\": \"2019-06-29T15:28:52.714Z\",\n - \ \"storageClass\": \"STANDARD\",\n \"timeStorageClassUpdated\": \"2019-06-29T15:28:52.714Z\",\n - \ \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n \"mediaLink\": - \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a?generation=1561822132714573&alt=media\",\n - \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CM34sOuAj+MCEAE=\"\n }\n ]\n}\n" + \"1565971690971479\",\n \"metageneration\": \"1\",\n \"timeCreated\": + \"2019-08-16T16:08:10.971Z\",\n \"updated\": \"2019-08-16T16:08:10.971Z\",\n + \ \"storageClass\": \"MULTI_REGIONAL\",\n \"timeStorageClassUpdated\": + \"2019-08-16T16:08:10.971Z\",\n \"size\": \"0\",\n \"md5Hash\": \"1B2M2Y8AsgTpgAmY7PhCfg==\",\n + \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/a?generation=1565971690971479&alt=media\",\n + \ \"crc32c\": \"AAAAAA==\",\n \"etag\": \"CNfC+ZLjh+QCEAE=\"\n }\n ]\n}\n" headers: Cache-Control: - private, max-age=0, must-revalidate, no-transform Content-Length: - - '754' + - '756' Content-Type: - application/json; charset=UTF-8 Server: @@ -598,7 +602,7 @@ interactions: Content-ID: - DELETE /storage/v1/b/gcsfs-testing/o/a%2Fb HTTP/1.1 + DELETE /storage/v1/b/gcsfs-testing/o/a HTTP/1.1 Content-Type: application/json @@ -616,7 +620,7 @@ interactions: Content-ID: - DELETE /storage/v1/b/gcsfs-testing/o/a%2Fc HTTP/1.1 + DELETE /storage/v1/b/gcsfs-testing/o/a%2Fb HTTP/1.1 Content-Type: application/json @@ -634,7 +638,7 @@ interactions: Content-ID: - DELETE /storage/v1/b/gcsfs-testing/o/a HTTP/1.1 + DELETE /storage/v1/b/gcsfs-testing/o/a%2Fc HTTP/1.1 Content-Type: application/json @@ -652,7 +656,7 @@ interactions: Connection: - keep-alive Content-Length: - - '892' + - '889' Content-Type: - multipart/mixed; boundary="===============7330845974216740156==" method: POST @@ -660,17 +664,17 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAN2PUWvCMBSF3wP5D/fd3bU3JltTZOBUtMJEWH3wqSRt0EJJw4yw/XsrCP4DhT2f - c77Dh2hNrI/Ve6X0rutEe9hX0+ni8CuL5edswdms99H5iOVfcDmYELq2NrHtfXKMMdzzYp7D5Med - Qu9PDq3QtWqcwLSxEqVOLVopCN+MIqt0ZhuiEX1wxtmqLLcJvRKIVMKmhxuQs7mJw+O3iS8gNKzP - fmiQBlK5yHIlYflVXvec4RMdxD9wGD/GAZGzC08bVXxvAgAA + H4sIAAAAAAAAAN2PyQrCMBRF94H8Q/b6bF6aaltEKIoD4rCoa0naqAVJQo0L/94Kgn+g4O7CnTgA + WoXqckx36VydvFwfqmNRTNHqUcxbR8nU2WBsgPLhTc6U99emUqFxNrqE4D/+apazcWtu3tmbAS2y + KqmNAF5rCTLjGrQUCEOVoE6yVNeIPZxQQsmyLPcRDpAJLtnWsfcgJTMVusd52/QZDllxP3cJzDqd + 8zTHmC025atPCfyQQfwBQ/wdBgBKnulyfKhvAgAA headers: Cache-Control: - private, max-age=0 Content-Encoding: - gzip Content-Type: - - multipart/mixed; boundary=batch_7_59Ull2igY_AAEgx4IGBCE + - multipart/mixed; boundary=batch_8O8Fafp4KUc_AAC1nb730ro Server: - GSE Transfer-Encoding: @@ -734,69 +738,78 @@ interactions: \"https://www.googleapis.com/storage/v1/b/anaconda-enterprise\",\n \"projectNumber\": \"586241054156\",\n \"name\": \"anaconda-enterprise\",\n \"timeCreated\": \"2017-07-05T23:53:06.552Z\",\n \"updated\": \"2017-07-14T17:39:54.178Z\",\n - \ \"metageneration\": \"3\",\n \"location\": \"US\",\n \"storageClass\": - \"MULTI_REGIONAL\",\n \"etag\": \"CAM=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"anaconda-public-data\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/anaconda-public-data\",\n + \ \"metageneration\": \"3\",\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"MULTI_REGIONAL\",\n \"etag\": + \"CAM=\"\n },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"anaconda-public-data\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/anaconda-public-data\",\n \ \"projectNumber\": \"586241054156\",\n \"name\": \"anaconda-public-data\",\n \ \"timeCreated\": \"2017-04-05T20:22:12.865Z\",\n \"updated\": \"2017-07-10T16:32:07.980Z\",\n - \ \"metageneration\": \"2\",\n \"location\": \"US\",\n \"storageClass\": - \"MULTI_REGIONAL\",\n \"etag\": \"CAI=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"artifacts.test_project.appspot.com\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/artifacts.test_project.appspot.com\",\n + \ \"metageneration\": \"2\",\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"MULTI_REGIONAL\",\n \"etag\": + \"CAI=\"\n },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"artifacts.test_project.appspot.com\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/artifacts.test_project.appspot.com\",\n \ \"projectNumber\": \"586241054156\",\n \"name\": \"artifacts.test_project.appspot.com\",\n \ \"timeCreated\": \"2016-05-17T18:29:22.774Z\",\n \"updated\": \"2018-10-29T22:44:41.802Z\",\n - \ \"metageneration\": \"3\",\n \"location\": \"US\",\n \"storageClass\": - \"STANDARD\",\n \"etag\": \"CAM=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"test_project_cloudbuild\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/test_project_cloudbuild\",\n + \ \"metageneration\": \"3\",\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"STANDARD\",\n \"etag\": \"CAM=\"\n + \ },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"test_project_cloudbuild\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/test_project_cloudbuild\",\n \ \"projectNumber\": \"586241054156\",\n \"name\": \"test_project_cloudbuild\",\n \ \"timeCreated\": \"2017-11-03T20:06:49.744Z\",\n \"updated\": \"2017-11-03T20:06:49.744Z\",\n - \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"storageClass\": - \"STANDARD\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"dataflow-anaconda-compute\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/dataflow-anaconda-compute\",\n + \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"STANDARD\",\n \"etag\": \"CAE=\"\n + \ },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"dataflow-anaconda-compute\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/dataflow-anaconda-compute\",\n \ \"projectNumber\": \"586241054156\",\n \"name\": \"dataflow-anaconda-compute\",\n \ \"timeCreated\": \"2017-09-14T18:55:42.848Z\",\n \"updated\": \"2017-09-14T18:55:42.848Z\",\n - \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"storageClass\": - \"MULTI_REGIONAL\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"gcsfs-tester\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-tester\",\n - \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-tester\",\n + \ \"metageneration\": \"1\",\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"MULTI_REGIONAL\",\n \"etag\": + \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"gcsfs-testing\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-testing\",\n \ \"timeCreated\": \"2019-06-28T20:15:57.944Z\",\n \"updated\": \"2019-06-28T20:15:57.944Z\",\n \ \"metageneration\": \"1\",\n \"iamConfiguration\": {\n \"bucketPolicyOnly\": {\n \"enabled\": true,\n \"lockedTime\": \"2019-09-26T20:15:57.944Z\"\n \ },\n \"uniformBucketLevelAccess\": {\n \"enabled\": true,\n \"lockedTime\": - \"2019-09-26T20:15:57.944Z\"\n }\n },\n \"location\": \"US\",\n \"storageClass\": - \"MULTI_REGIONAL\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"gcsfs-testing\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing\",\n - \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-testing\",\n + \"2019-09-26T20:15:57.944Z\"\n }\n },\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"MULTI_REGIONAL\",\n \"etag\": + \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"gcsfs-testing2\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing2\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-testing2\",\n \ \"timeCreated\": \"2019-06-29T02:18:56.542Z\",\n \"updated\": \"2019-06-29T02:18:56.542Z\",\n \ \"metageneration\": \"1\",\n \"iamConfiguration\": {\n \"bucketPolicyOnly\": {\n \"enabled\": false\n },\n \"uniformBucketLevelAccess\": {\n - \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"storageClass\": - \"STANDARD\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"gcsfs-tester3\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-tester3\",\n - \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-tester3\",\n + \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"STANDARD\",\n \"etag\": \"CAE=\"\n + \ },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"gcsfs-testing3\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing3\",\n + \ \"projectNumber\": \"586241054156\",\n \"name\": \"gcsfs-testing3\",\n \ \"timeCreated\": \"2019-06-28T20:14:01.241Z\",\n \"updated\": \"2019-06-28T20:14:01.241Z\",\n \ \"metageneration\": \"1\",\n \"iamConfiguration\": {\n \"bucketPolicyOnly\": {\n \"enabled\": false\n },\n \"uniformBucketLevelAccess\": {\n - \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"storageClass\": - \"STANDARD\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"kubeflow-anaconda-test\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/kubeflow-anaconda-test\",\n + \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"STANDARD\",\n \"etag\": \"CAE=\"\n + \ },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"kubeflow-anaconda-test\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/kubeflow-anaconda-test\",\n \ \"projectNumber\": \"586241054156\",\n \"name\": \"kubeflow-anaconda-test\",\n \ \"timeCreated\": \"2018-11-12T18:18:30.094Z\",\n \"updated\": \"2018-11-12T18:18:30.094Z\",\n \ \"metageneration\": \"1\",\n \"iamConfiguration\": {\n \"bucketPolicyOnly\": {\n \"enabled\": false\n },\n \"uniformBucketLevelAccess\": {\n - \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"storageClass\": - \"MULTI_REGIONAL\",\n \"etag\": \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n - \ \"id\": \"mdtemp\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/mdtemp\",\n - \ \"projectNumber\": \"586241054156\",\n \"name\": \"mdtemp\",\n \"timeCreated\": - \"2018-04-10T01:17:29.182Z\",\n \"updated\": \"2018-05-10T16:59:01.351Z\",\n - \ \"metageneration\": \"2\",\n \"iamConfiguration\": {\n \"bucketPolicyOnly\": - {\n \"enabled\": false\n },\n \"uniformBucketLevelAccess\": {\n - \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"storageClass\": - \"STANDARD\",\n \"etag\": \"CAI=\"\n }\n ]\n}\n" + \ \"enabled\": false\n }\n },\n \"location\": \"US\",\n \"locationType\": + \"multi-region\",\n \"storageClass\": \"MULTI_REGIONAL\",\n \"etag\": + \"CAE=\"\n },\n {\n \"kind\": \"storage#bucket\",\n \"id\": \"mdtemp\",\n + \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/mdtemp\",\n \"projectNumber\": + \"586241054156\",\n \"name\": \"mdtemp\",\n \"timeCreated\": \"2018-04-10T01:17:29.182Z\",\n + \ \"updated\": \"2018-05-10T16:59:01.351Z\",\n \"metageneration\": \"2\",\n + \ \"iamConfiguration\": {\n \"bucketPolicyOnly\": {\n \"enabled\": + false\n },\n \"uniformBucketLevelAccess\": {\n \"enabled\": false\n + \ }\n },\n \"location\": \"US\",\n \"locationType\": \"multi-region\",\n + \ \"storageClass\": \"STANDARD\",\n \"etag\": \"CAI=\"\n }\n ]\n}\n" headers: Cache-Control: - private, max-age=0, must-revalidate, no-transform Content-Length: - - '4949' + - '5299' Content-Type: - application/json; charset=UTF-8 Server: diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index cfb96653..dfa037cb 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -209,7 +209,7 @@ def test_gcs_glob(token_restore): assert fn in gcs.glob(TEST_BUCKET+'/*/*') assert fn in gcs.glob(TEST_BUCKET+'/**') assert all(f in gcs.find(TEST_BUCKET) for f in - gcs.glob(TEST_BUCKET+'/nested/*')) + gcs.glob(TEST_BUCKET+'/nested/*') if gcs.isfile(f)) @my_vcr.use_cassette(match=['all']) diff --git a/gcsfs/tests/test_fuse.py b/gcsfs/tests/test_fuse.py index ae21318b..3be4acaa 100644 --- a/gcsfs/tests/test_fuse.py +++ b/gcsfs/tests/test_fuse.py @@ -11,9 +11,6 @@ import time -PY2 = sys.version_info.major < 3 - - @pytest.mark.xfail @pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") diff --git a/gcsfs/tests/test_mapping.py b/gcsfs/tests/test_mapping.py index dc6dbca6..f557d4be 100644 --- a/gcsfs/tests/test_mapping.py +++ b/gcsfs/tests/test_mapping.py @@ -132,7 +132,7 @@ def test_new_bucket(token_restore): pass with pytest.raises(Exception) as e: d = gcs.get_mapper(new_bucket, check=True) - assert 'create=True' in str(e) + assert 'create=True' in str(e.value) try: d = gcs.get_mapper(new_bucket, create=True)