From f45e48a6500a1d1e239be06142150beb0eeb44ed Mon Sep 17 00:00:00 2001 From: Addison Schiller Date: Wed, 4 Oct 2017 16:31:20 -0400 Subject: [PATCH 1/2] Add size-cast-as-int property to file metadata. * Owncloud has the unfortunate habit of returning file size as a string instead of an int. WB never enforced a cast to int on the property, so there may be clients in the wild that expect it to be a string. To avoid breaking these, add a new property, `sizeInt` to WB metadata responses. This is guaranteed to be either an `int` or `None` if the size is unknown. * The JSON-API -style responses for folder metadata include a `size` field that is always `None`. Add a similar `sizeInt` field for parity with files. * Update explicit metadata tests for all providers. * Update type annotation for BaseFileMetadata.size to reflect its regrettable potential for stringiness. --- tests/core/test_metadata.py | 3 +++ tests/providers/bitbucket/test_metadata.py | 1 + tests/providers/box/test_metadata.py | 9 +++++++-- tests/providers/cloudfiles/test_metadata.py | 15 ++++++++++++--- tests/providers/dataverse/test_metadata.py | 2 ++ tests/providers/dropbox/test_metadata.py | 8 +++++++- tests/providers/figshare/test_metadata.py | 10 ++++++++-- tests/providers/filesystem/test_metadata.py | 15 +++++++++++---- tests/providers/filesystem/test_provider.py | 4 ++++ tests/providers/github/test_metadata.py | 6 ++++++ tests/providers/gitlab/test_provider.py | 1 + tests/providers/googlecloud/test_metadata.py | 1 + tests/providers/googledrive/test_metadata.py | 4 ++++ tests/providers/onedrive/test_metadata.py | 1 + tests/providers/osfstorage/test_metadata.py | 1 + tests/providers/owncloud/test_metadata.py | 3 +++ tests/providers/owncloud/test_provider.py | 2 ++ tests/providers/s3/test_metadata.py | 4 ++++ tests/tasks/test_copy.py | 1 + tests/tasks/test_move.py | 1 + waterbutler/core/metadata.py | 19 +++++++++++++++++-- 21 files changed, 97 insertions(+), 14 deletions(-) diff --git a/tests/core/test_metadata.py b/tests/core/test_metadata.py index 5f3173b7f..d072e2cbd 100644 --- a/tests/core/test_metadata.py +++ b/tests/core/test_metadata.py @@ -38,6 +38,7 @@ def test_file_json_api_serialize(self): 'modified_utc': 'never', 'created_utc': 'always', 'size': 1337, + 'sizeInt': 1337, 'resource': 'n0d3z', } assert 'new_folder' not in serialized['links'] @@ -63,6 +64,7 @@ def test_folder_json_api_serialize(self): 'materialized': '/Bar/', 'etag': etag, 'size': None, + 'sizeInt': None, 'resource': 'n0d3z', } assert serialized['links']['new_folder'].endswith(link_suffix + '?kind=folder') @@ -92,6 +94,7 @@ def test_folder_json_api_size_serialize(self): 'modified_utc': 'never', 'created_utc': 'always', 'size': 1337, + 'sizeInt': 1337, } def test_file_revision_json_api_serialize(self): diff --git a/tests/providers/bitbucket/test_metadata.py b/tests/providers/bitbucket/test_metadata.py index 6de003423..9228cc776 100644 --- a/tests/providers/bitbucket/test_metadata.py +++ b/tests/providers/bitbucket/test_metadata.py @@ -35,6 +35,7 @@ def test_build_file_metadata(self, file_metadata, owner, repo): assert metadata.created_utc is None assert metadata.content_type is None assert metadata.size == 13 + assert metadata.size_as_int == 13 assert metadata.etag == '{}::{}'.format(full_path,COMMIT_SHA) assert metadata.provider == 'bitbucket' assert metadata.last_commit_sha == '90c8f7eef948' diff --git a/tests/providers/box/test_metadata.py b/tests/providers/box/test_metadata.py index b01d2ad5f..fed522d67 100644 --- a/tests/providers/box/test_metadata.py +++ b/tests/providers/box/test_metadata.py @@ -20,6 +20,8 @@ def test_file_metadata(self, root_provider_fixtures): assert data.path == '/5000948880' assert data.provider == 'box' assert data.size == 629644 + assert data.size_as_int == 629644 + assert type(data.size_as_int) == int assert data.modified == '2012-12-12T11:04:26-08:00' assert data.created_utc == '2012-12-12T18:55:30+00:00' assert data.content_type is None @@ -48,7 +50,8 @@ def test_file_metadata(self, root_provider_fixtures): 'modified': '2012-12-12T11:04:26-08:00', 'modified_utc': '2012-12-12T19:04:26+00:00', 'created_utc': '2012-12-12T18:55:30+00:00', - 'size': 629644 + 'size': 629644, + 'sizeInt': 629644, } assert data.kind == 'file' assert data.modified_utc == '2012-12-12T19:04:26+00:00' @@ -72,6 +75,7 @@ def test_file_metadata(self, root_provider_fixtures): 'modified_utc': '2012-12-12T19:04:26+00:00', 'created_utc': '2012-12-12T18:55:30+00:00', 'size': 629644, + 'sizeInt': 629644, 'resource': 'cn42d' }, 'links': { @@ -113,7 +117,8 @@ def test_folder_metadata(self, intra_fixtures): 'materialized': '/moveablefolder/', 'etag': '299a515e98fe1c548c6fe6141d01a43e739cf965b0d324aa0961924162f7af79', 'resource': '7ycmyr', - 'size': None + 'size': None, + 'sizeInt': None, }, 'links': { 'move': 'http://localhost:7777/v1/resources/7ycmyr/providers/box/36833297084/', diff --git a/tests/providers/cloudfiles/test_metadata.py b/tests/providers/cloudfiles/test_metadata.py index b69e44cc3..30aefa139 100644 --- a/tests/providers/cloudfiles/test_metadata.py +++ b/tests/providers/cloudfiles/test_metadata.py @@ -43,6 +43,8 @@ def test_header_metadata(self, file_header_metadata_txt): assert data.path == '/file.txt' assert data.provider == 'cloudfiles' assert data.size == 216945 + assert type(data.size_as_int) == int + assert data.size_as_int == 216945 assert data.modified == 'Mon, 22 Dec 2014 19:01:02 GMT' assert data.created_utc is None assert data.content_type == 'text/plain' @@ -69,7 +71,8 @@ def test_header_metadata(self, file_header_metadata_txt): 'modified': 'Mon, 22 Dec 2014 19:01:02 GMT', 'modified_utc': '2014-12-22T19:01:02+00:00', 'created_utc': None, - 'size': 216945 + 'size': 216945, + 'sizeInt': 216945, } assert data.json_api_serialized('cn42d') == { @@ -92,6 +95,7 @@ def test_header_metadata(self, file_header_metadata_txt): 'modified_utc': '2014-12-22T19:01:02+00:00', 'created_utc': None, 'size': 216945, + 'sizeInt': 216945, 'resource': 'cn42d' }, 'links': { @@ -117,6 +121,8 @@ def test_file_metadata(self, file_metadata): assert data.provider == 'cloudfiles' assert data.path == '/similar.file' assert data.size == 190 + assert data.size_as_int == 190 + assert type(data.size_as_int) == int assert data.modified == '2014-12-19T23:22:14.728640' assert data.created_utc is None assert data.content_type == 'application/x-www-form-urlencoded;charset=utf-8' @@ -142,7 +148,8 @@ def test_file_metadata(self, file_metadata): 'modified': '2014-12-19T23:22:14.728640', 'modified_utc': '2014-12-19T23:22:14+00:00', 'created_utc': None, - 'size': 190 + 'size': 190, + 'sizeInt': 190, } assert data.json_api_serialized('cn42d') == { @@ -165,6 +172,7 @@ def test_file_metadata(self, file_metadata): 'modified_utc': '2014-12-19T23:22:14+00:00', 'created_utc': None, 'size': 190, + 'sizeInt': 190, 'resource': 'cn42d' }, 'links': { @@ -222,7 +230,8 @@ def test_folder_metadata(self): 'materialized': '/level1/', 'etag': '69cf764abe6f2e90dc81fb4218e15f202f9b99bcd1963cf2d5f011629d6f0d8a', 'resource': 'cn42d', - 'size': None + 'size': None, + 'sizeInt': None }, 'links': { 'move': 'http://localhost:7777/v1/resources/cn42d/providers/cloudfiles/level1/', diff --git a/tests/providers/dataverse/test_metadata.py b/tests/providers/dataverse/test_metadata.py index ccb139087..c9c17900e 100644 --- a/tests/providers/dataverse/test_metadata.py +++ b/tests/providers/dataverse/test_metadata.py @@ -38,6 +38,8 @@ def test_file_metadata(self, file_metadata_object): assert file_metadata_object.kind == 'file' assert file_metadata_object.file_id == '20' assert file_metadata_object.name == 'thefile.txt' + assert file_metadata_object.size is None + assert file_metadata_object.size_as_int is None assert file_metadata_object.path == '/20' assert file_metadata_object.materialized_path == '/thefile.txt' assert not file_metadata_object.size diff --git a/tests/providers/dropbox/test_metadata.py b/tests/providers/dropbox/test_metadata.py index 55684d28a..27dd75abe 100644 --- a/tests/providers/dropbox/test_metadata.py +++ b/tests/providers/dropbox/test_metadata.py @@ -15,6 +15,9 @@ def test_file_metadata(self, provider_fixtures): assert data.name == 'Getting_Started.pdf' assert data.path == '/Getting_Started.pdf' assert data.size == 124778 + assert data.size_as_int == 124778 + assert type(data.size_as_int) == int + assert data.modified == '2016-06-13T19:08:17Z' assert data.created_utc is None assert data.content_type is None @@ -40,7 +43,8 @@ def test_file_metadata(self, provider_fixtures): 'modified': '2016-06-13T19:08:17Z', 'modified_utc': '2016-06-13T19:08:17+00:00', 'created_utc': None, - 'size': 124778 + 'size': 124778, + 'sizeInt': 124778, } assert data.kind == 'file' assert data.materialized_path == '/Getting_Started.pdf' @@ -75,6 +79,7 @@ def test_file_metadata(self, provider_fixtures): 'modified_utc': '2016-06-13T19:08:17+00:00', 'created_utc': None, 'size': 124778, + 'sizeInt': 124778, 'resource': 'jverwz' }, 'links': { @@ -115,6 +120,7 @@ def test_folder_metadata(self, provider_fixtures): 'etag': 'bbd6cc654c4a3ca1124b69fccb392ec9754e18e9094effb525192509f8e1b901', 'resource': 'mucuew', 'size': None, + 'sizeInt': None, }, 'links': { 'move': link_url, diff --git a/tests/providers/figshare/test_metadata.py b/tests/providers/figshare/test_metadata.py index 1899d1f0c..d25259c4a 100644 --- a/tests/providers/figshare/test_metadata.py +++ b/tests/providers/figshare/test_metadata.py @@ -24,6 +24,9 @@ def test_file_metadata(self, root_provider_fixtures): assert data.materialized_path == '/file_article/file' assert data.upload_path == '/4037952/6530715' assert data.size == 7 + assert data.size_as_int == 7 + assert type(data.size_as_int) == int + assert data.content_type is None assert data.modified is None assert data.modified_utc is None @@ -67,7 +70,8 @@ def test_file_metadata(self, root_provider_fixtures): 'modified': None, 'modified_utc': None, 'created_utc': None, - 'size': 7 + 'size': 7, + 'sizeInt': 7, } api_url = 'http://localhost:7777/v1/resources/cn42d/providers/figshare/4037952/6530715' @@ -97,6 +101,7 @@ def test_file_metadata(self, root_provider_fixtures): 'modified_utc': None, 'created_utc': None, 'size': 7, + 'sizeInt': 7, 'resource': 'cn42d' }, 'links': { @@ -181,7 +186,8 @@ def test_folder_metadata(self, root_provider_fixtures): 'materialized': '/folder_article/', 'etag': '6bef522e6f14597fd939b6b5c29e99091dc0b0badcac332da6e75bec0a69cf5e', 'resource': '45hjnz', - 'size': None + 'size': None, + 'sizeInt': None, }, 'links': { 'move': api_url, diff --git a/tests/providers/filesystem/test_metadata.py b/tests/providers/filesystem/test_metadata.py index 7524e0ffc..afd467071 100644 --- a/tests/providers/filesystem/test_metadata.py +++ b/tests/providers/filesystem/test_metadata.py @@ -47,6 +47,8 @@ def test_file_metadata(self, file_metadata): assert data.content_type is None assert data.name == '77094244-aa24-48da-9437-d8ce6f7a94e9' assert data.size == 35981 + assert data.size_as_int == 35981 + assert type(data.size_as_int) == int assert data.etag == ('Wed, 20 Sep 2017 15:16:02 +0000::/' 'code/website/osfstoragecache/77094244-aa24-48da-9437-d8ce6f7a94e9') assert data.kind == 'file' @@ -63,7 +65,8 @@ def test_file_metadata(self, file_metadata): 'modified': 'Wed, 20 Sep 2017 15:16:02 +0000', 'modified_utc': '2017-09-20T15:16:02.601916+00:00', 'created_utc': None, - 'size': 35981 + 'size': 35981, + 'sizeInt': 35981 } assert data.json_api_serialized('cn42d') == { @@ -83,6 +86,7 @@ def test_file_metadata(self, file_metadata): 'modified_utc': '2017-09-20T15:16:02.601916+00:00', 'created_utc': None, 'size': 35981, + 'sizeInt': 35981, 'resource': 'cn42d' }, 'links': { @@ -141,7 +145,8 @@ def test_root_metadata(self, root_metadata): 'materialized': '/', 'etag': '6a2b72b88f67692ff6f4cc3a52798cdc54a6e7c7e6dcbf8463fcb5105b6b949e', 'resource': '7ycmyr', - 'size': None + 'size': None, + 'sizeInt': None, }, 'links': { 'move': 'http://localhost:7777/v1/resources/7ycmyr/providers/filesystem/', @@ -195,7 +200,8 @@ def test_folder_metadata(self, folder_metadata): 'materialized': '/folder1/', 'etag': '6a2b72b88f67692ff6f4cc3a52798cdc54a6e7c7e6dcbf8463fcb5105b6b949e', 'resource': '7ycmyr', - 'size': None + 'size': None, + 'sizeInt': None, }, 'links': { 'move': 'http://localhost:7777/v1/resources/7ycmyr/providers/filesystem/folder1/', @@ -249,7 +255,8 @@ def test_subfolder_metadata(self, subfolder_metadata): 'materialized': '/folder1/folder2/', 'etag': '6a2b72b88f67692ff6f4cc3a52798cdc54a6e7c7e6dcbf8463fcb5105b6b949e', 'resource': '7ycmyr', - 'size': None + 'size': None, + 'sizeInt': None }, 'links': { 'move': 'http://localhost:7777/v1/resources/7ycmyr/providers/filesystem/' diff --git a/tests/providers/filesystem/test_provider.py b/tests/providers/filesystem/test_provider.py index 50a332725..132358a5d 100644 --- a/tests/providers/filesystem/test_provider.py +++ b/tests/providers/filesystem/test_provider.py @@ -140,6 +140,7 @@ async def test_upload_create(self, provider): assert metadata.name == file_name assert metadata.path == file_path assert metadata.size == len(file_content) + assert metadata.size_as_int == len(file_content) assert created is True @pytest.mark.asyncio @@ -156,6 +157,7 @@ async def test_upload_update(self, provider): assert metadata.name == file_name assert metadata.path == file_path assert metadata.size == len(file_content) + assert metadata.size_as_int == len(file_content) assert created is False @pytest.mark.asyncio @@ -172,6 +174,7 @@ async def test_upload_nested_create(self, provider): assert metadata.name == file_name assert metadata.path == file_path assert metadata.size == len(file_content) + assert metadata.size_as_int == len(file_content) assert created is True @pytest.mark.asyncio @@ -188,6 +191,7 @@ async def test_upload_nested_update(self, provider): assert metadata.name == file_name assert metadata.path == file_path assert metadata.size == len(file_content) + assert metadata.size_as_int == len(file_content) assert created is False @pytest.mark.asyncio diff --git a/tests/providers/github/test_metadata.py b/tests/providers/github/test_metadata.py index cc89d7e49..2d1ef358b 100644 --- a/tests/providers/github/test_metadata.py +++ b/tests/providers/github/test_metadata.py @@ -25,6 +25,8 @@ def test_build_file_metadata_from_tree(self, metadata_fixtures): assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 38 + assert metadata.size_as_int == 38 + assert type(metadata.size_as_int) == int assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341' assert metadata.extra == { 'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341', @@ -50,6 +52,8 @@ def test_build_file_metadata_from_contents(self, metadata_fixtures): assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 15 + assert metadata.size_as_int == 15 + assert type(metadata.size_as_int) == int assert metadata.etag == '/epsilon::bd4fb614678f544acb22bac6861a21108f1e5d10' assert metadata.extra == { 'fileSha': 'bd4fb614678f544acb22bac6861a21108f1e5d10', @@ -103,6 +107,8 @@ def test_file_metadata_with_ref(self, metadata_fixtures): assert metadata.modified is None assert metadata.content_type is None assert metadata.size == 38 + assert metadata.size_as_int == 38 + assert type(metadata.size_as_int) == int assert metadata.etag == '/README.md::d863d70539aa9fcb6b44b057221706f2ab18e341' assert metadata.extra == { 'fileSha': 'd863d70539aa9fcb6b44b057221706f2ab18e341', diff --git a/tests/providers/gitlab/test_provider.py b/tests/providers/gitlab/test_provider.py index 95d2baf96..8e4c20781 100644 --- a/tests/providers/gitlab/test_provider.py +++ b/tests/providers/gitlab/test_provider.py @@ -277,6 +277,7 @@ async def test_metadata_file_with_default_ref(self, provider): 'name': 'file', 'kind': 'file', 'size': 123, + 'sizeInt': 123, 'provider':'gitlab', 'path': path, 'materialized': path, diff --git a/tests/providers/googlecloud/test_metadata.py b/tests/providers/googlecloud/test_metadata.py index 6504ed44b..e738a1396 100644 --- a/tests/providers/googlecloud/test_metadata.py +++ b/tests/providers/googlecloud/test_metadata.py @@ -93,6 +93,7 @@ def test_file_metadata(self, file_name, file_obj_name, meta_file_parsed, meta_fi assert metadata.created_utc is None assert metadata.etag == '9a46947c9c622d7792125d8ea44c4638' assert metadata.size == 85 + assert metadata.size_as_int == 85 assert metadata.extra == dict(metadata_extra) diff --git a/tests/providers/googledrive/test_metadata.py b/tests/providers/googledrive/test_metadata.py index f425504ea..adc611c07 100644 --- a/tests/providers/googledrive/test_metadata.py +++ b/tests/providers/googledrive/test_metadata.py @@ -32,6 +32,8 @@ def test_file_metadata_drive(self, basepath, root_provider_fixtures): assert parsed.id == item['id'] assert path.name == item['title'] assert parsed.name == item['title'] + assert parsed.size_as_int == 918668 + assert type(parsed.size_as_int) == int assert parsed.size == item['fileSize'] assert parsed.modified == item['modifiedDate'] assert parsed.content_type == item['mimeType'] @@ -55,6 +57,8 @@ def test_file_metadata_drive_slashes(self, basepath, root_provider_fixtures): assert parsed.name == item['title'] assert parsed.name == path.name assert parsed.size == item['fileSize'] + assert parsed.size_as_int == 918668 + assert type(parsed.size_as_int) == int assert parsed.modified == item['modifiedDate'] assert parsed.content_type == item['mimeType'] assert parsed.extra == { diff --git a/tests/providers/onedrive/test_metadata.py b/tests/providers/onedrive/test_metadata.py index c2b599c63..6278a46f2 100644 --- a/tests/providers/onedrive/test_metadata.py +++ b/tests/providers/onedrive/test_metadata.py @@ -26,6 +26,7 @@ def test_build_file_metadata(self, root_provider_fixtures): assert metadata.name == 'toes.txt' assert metadata.path == '/{}'.format(root_provider_fixtures['file_id']) assert metadata.size == 11 + assert metadata.size_as_int == 11 assert metadata.modified == '2017-08-17T17:49:50.38Z' assert metadata.modified_utc == '2017-08-17T17:49:50+00:00' assert metadata.content_type == 'text/plain' diff --git a/tests/providers/osfstorage/test_metadata.py b/tests/providers/osfstorage/test_metadata.py index 5234ac490..9c56bbd93 100644 --- a/tests/providers/osfstorage/test_metadata.py +++ b/tests/providers/osfstorage/test_metadata.py @@ -17,6 +17,7 @@ def test_file_metadata(self, file_metadata, file_metadata_object): assert file_metadata_object.path == '/59a9b628b7d1c903ab5a8f52' assert str(file_metadata_object.materialized_path) == '/doc.rst' assert file_metadata_object.size == 5596 + assert file_metadata_object.size_as_int == 5596 assert file_metadata_object.modified == '2017-09-01T19:34:00.175741+00:00' assert file_metadata_object.kind == 'file' assert file_metadata_object.content_type == None diff --git a/tests/providers/owncloud/test_metadata.py b/tests/providers/owncloud/test_metadata.py index 4d919f88d..e66978bec 100644 --- a/tests/providers/owncloud/test_metadata.py +++ b/tests/providers/owncloud/test_metadata.py @@ -21,6 +21,8 @@ def test_file_metadata(self, file_metadata_object): assert file_metadata_object.materialized_path == '/Documents/dissertation.aux' assert file_metadata_object.kind == 'file' assert file_metadata_object.size == '3011' + assert file_metadata_object.size_as_int == 3011 + assert type(file_metadata_object.size_as_int) == int assert file_metadata_object.etag == '"a3c411808d58977a9ecd7485b5b7958e"' assert file_metadata_object.modified == 'Sun, 10 Jul 2016 23:28:31 GMT' assert file_metadata_object.modified_utc == '2016-07-10T23:28:31+00:00' @@ -47,6 +49,7 @@ def test_file_metadata_less_info(self, file_metadata_object_less_info): assert file_metadata_object_less_info.materialized_path == '/Documents/dissertation.aux' assert file_metadata_object_less_info.kind == 'file' assert file_metadata_object_less_info.size is None + assert file_metadata_object_less_info.size_as_int is None assert file_metadata_object_less_info.etag == '"a3c411808d58977a9ecd7485b5b7958e"' assert file_metadata_object_less_info.modified == 'Sun, 10 Jul 2016 23:28:31 GMT' assert file_metadata_object_less_info.modified_utc == '2016-07-10T23:28:31+00:00' diff --git a/tests/providers/owncloud/test_provider.py b/tests/providers/owncloud/test_provider.py index f35e2d065..d83a33479 100644 --- a/tests/providers/owncloud/test_provider.py +++ b/tests/providers/owncloud/test_provider.py @@ -190,6 +190,7 @@ async def test_upload(self, provider, file_stream, file_metadata, file_metadata_ assert created is True assert metadata.name == file_metadata_object.name assert metadata.size == file_metadata_object.size + assert metadata.size_as_int == int(file_metadata_object.size) assert aiohttpretty.has_call(method='PUT', uri=url) @pytest.mark.asyncio @@ -213,6 +214,7 @@ async def test_upload_keep(self, provider, file_stream, file_metadata, file_meta assert created is True assert metadata.name == file_metadata_object.name assert metadata.size == file_metadata_object.size + assert metadata.size_as_int == int(file_metadata_object.size) assert aiohttpretty.has_call(method='PUT', uri=url) @pytest.mark.asyncio diff --git a/tests/providers/s3/test_metadata.py b/tests/providers/s3/test_metadata.py index 2b2c68d92..85bac7eaf 100644 --- a/tests/providers/s3/test_metadata.py +++ b/tests/providers/s3/test_metadata.py @@ -21,6 +21,8 @@ def test_file_metadata_headers(self, file_metadata_headers_object, file_header_m assert file_metadata_headers_object.kind == 'file' assert file_metadata_headers_object.provider == 's3' assert file_metadata_headers_object.size == 9001 + assert file_metadata_headers_object.size_as_int == 9001 + assert type(file_metadata_headers_object.size_as_int) == int assert file_metadata_headers_object.content_type == 'binary/octet-stream' assert file_metadata_headers_object.modified == 'SomeTime' assert not file_metadata_headers_object.created_utc @@ -50,6 +52,8 @@ def test_file_metadata(self, file_metadata_object): assert file_metadata_object.path == '/my-image.jpg' assert file_metadata_object.materialized_path == '/my-image.jpg' assert file_metadata_object.size == 434234 + assert file_metadata_object.size_as_int == 434234 + assert type(file_metadata_object.size_as_int) == int assert file_metadata_object.modified == '2009-10-12T17:50:30.000Z' assert file_metadata_object.etag == 'fba9dede5f27731c9771645a39863328' assert not file_metadata_object.created_utc diff --git a/tests/tasks/test_copy.py b/tests/tasks/test_copy.py index e2d655bd9..15d0ee005 100644 --- a/tests/tasks/test_copy.py +++ b/tests/tasks/test_copy.py @@ -212,6 +212,7 @@ def test_return_values(self, event_loop, providers, bundles, callback, src_path, 'modified_utc': metadata.modified_utc, 'created_utc': metadata.created_utc, 'size': metadata.size, + 'sizeInt': metadata.size_as_int, } def test_starttime_override(self, event_loop, providers, bundles, callback, mock_time): diff --git a/tests/tasks/test_move.py b/tests/tasks/test_move.py index b998ea1a6..60483efcc 100644 --- a/tests/tasks/test_move.py +++ b/tests/tasks/test_move.py @@ -211,6 +211,7 @@ def test_return_values(self, event_loop, providers, bundles, callback, src_path, 'modified_utc': metadata.modified_utc, 'created_utc': metadata.created_utc, 'size': metadata.size, + 'sizeInt': metadata.size_as_int, } def test_starttime_override(self, event_loop, providers, bundles, callback, mock_time): diff --git a/waterbutler/core/metadata.py b/waterbutler/core/metadata.py index 97f3c3856..c5990563c 100644 --- a/waterbutler/core/metadata.py +++ b/waterbutler/core/metadata.py @@ -218,6 +218,7 @@ def serialized(self) -> dict: 'modified_utc': self.modified_utc, 'created_utc': self.created_utc, 'size': self.size, + 'sizeInt': self.size_as_int, }) def _json_api_links(self, resource: str) -> dict: @@ -261,10 +262,23 @@ def created_utc(self) -> str: @property @abc.abstractmethod - def size(self) -> int: - """ Size of the file in bytes. """ + def size(self) -> typing.Union[int, str]: + """ Size of the file in bytes. Should be a int, but some providers return a string and WB + never casted it. The `size_as_int` property was added to enforce this without breaking + exisiting code and workarounds. + """ raise NotImplementedError + @property + def size_as_int(self) -> int: + """ Size of the file in bytes. Always an `int` or `None`. Some providers report size as a + `str`. Both exist to maintain backwards compatibility. + """ + if self.size is not None: + return int(self.size) + else: + return None + class BaseFileRevisionMetadata(metaclass=abc.ABCMeta): @@ -354,6 +368,7 @@ def json_api_serialized(self, resource: str) -> dict: """ ret = super().json_api_serialized(resource) ret['attributes']['size'] = None + ret['attributes']['sizeInt'] = None return ret def _json_api_links(self, resource: str) -> dict: From 2f2f8bcb06c5bc2b91f8dd681ce0963a42c9e636 Mon Sep 17 00:00:00 2001 From: TomBaxter Date: Thu, 21 Dec 2017 15:27:23 -0500 Subject: [PATCH 2/2] Minor logic change SVCS-499 --- waterbutler/core/metadata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/waterbutler/core/metadata.py b/waterbutler/core/metadata.py index c5990563c..b6788c59d 100644 --- a/waterbutler/core/metadata.py +++ b/waterbutler/core/metadata.py @@ -274,10 +274,11 @@ def size_as_int(self) -> int: """ Size of the file in bytes. Always an `int` or `None`. Some providers report size as a `str`. Both exist to maintain backwards compatibility. """ - if self.size is not None: - return int(self.size) - else: - return None + try: + size_as_int = int(self.size) + except (TypeError, ValueError): + size_as_int = None + return size_as_int class BaseFileRevisionMetadata(metaclass=abc.ABCMeta):