From d5ec442d7bd5ebb2033e9a7b9ad327d30a2dde93 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Mon, 15 Apr 2024 17:31:39 +0100 Subject: [PATCH 1/5] Added logic to correctly move the properties metadata file if it's included in the export --- darwin/dataset/remote_dataset.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/darwin/dataset/remote_dataset.py b/darwin/dataset/remote_dataset.py index b4e28e960..7ec1d1039 100644 --- a/darwin/dataset/remote_dataset.py +++ b/darwin/dataset/remote_dataset.py @@ -278,6 +278,13 @@ def pull( annotations_dir.mkdir(parents=True, exist_ok=False) stems: dict = {} + # If properties were exported, move the manifest file to the annotations folder + if (tmp_dir / ".v7").exists(): + metadata_file = tmp_dir / ".v7" / "metadata.json" + metadata_dir = annotations_dir / ".v7" + metadata_dir.mkdir(parents=True, exist_ok=True) + shutil.move(str(metadata_file), str(metadata_dir / "metadata.json")) + # Move the annotations into the right folder and rename them to have the image # original filename as contained in the json for annotation_path in tmp_dir.glob("*.json"): From f542e1c028c6f11a98c78f523f829fcfc21ce66d Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Mon, 15 Apr 2024 17:33:50 +0100 Subject: [PATCH 2/5] Small fix to mkdir statement --- darwin/dataset/remote_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin/dataset/remote_dataset.py b/darwin/dataset/remote_dataset.py index 7ec1d1039..21e42bf78 100644 --- a/darwin/dataset/remote_dataset.py +++ b/darwin/dataset/remote_dataset.py @@ -282,7 +282,7 @@ def pull( if (tmp_dir / ".v7").exists(): metadata_file = tmp_dir / ".v7" / "metadata.json" metadata_dir = annotations_dir / ".v7" - metadata_dir.mkdir(parents=True, exist_ok=True) + metadata_dir.mkdir(parents=True, exist_ok=False) shutil.move(str(metadata_file), str(metadata_dir / "metadata.json")) # Move the annotations into the right folder and rename them to have the image From b2fe02bc33b1347e4ba05d49f0ad73b2f7ee1801 Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:20:43 +0100 Subject: [PATCH 3/5] Update darwin/dataset/remote_dataset.py Co-authored-by: saurbhc --- darwin/dataset/remote_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin/dataset/remote_dataset.py b/darwin/dataset/remote_dataset.py index 21e42bf78..a6d288769 100644 --- a/darwin/dataset/remote_dataset.py +++ b/darwin/dataset/remote_dataset.py @@ -278,7 +278,7 @@ def pull( annotations_dir.mkdir(parents=True, exist_ok=False) stems: dict = {} - # If properties were exported, move the manifest file to the annotations folder + # If properties were exported, move the metadata.json file to the annotations folder if (tmp_dir / ".v7").exists(): metadata_file = tmp_dir / ".v7" / "metadata.json" metadata_dir = annotations_dir / ".v7" From 735cdd3cf18ab8af45e5f363a5c32f7451b49fd8 Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:27:03 +0100 Subject: [PATCH 4/5] Update darwin/dataset/remote_dataset.py Co-authored-by: saurbhc --- darwin/dataset/remote_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin/dataset/remote_dataset.py b/darwin/dataset/remote_dataset.py index a6d288769..0be301c22 100644 --- a/darwin/dataset/remote_dataset.py +++ b/darwin/dataset/remote_dataset.py @@ -282,7 +282,7 @@ def pull( if (tmp_dir / ".v7").exists(): metadata_file = tmp_dir / ".v7" / "metadata.json" metadata_dir = annotations_dir / ".v7" - metadata_dir.mkdir(parents=True, exist_ok=False) + metadata_dir.mkdir(parents=True, exist_ok=True) shutil.move(str(metadata_file), str(metadata_dir / "metadata.json")) # Move the annotations into the right folder and rename them to have the image From df270d01749104790cb7c6d1fd78d2b501a36e3a Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Wed, 17 Apr 2024 10:11:08 +0100 Subject: [PATCH 5/5] Added test case --- tests/darwin/dataset/remote_dataset_test.py | 38 ++++++++++++++++++++ tests/dataset_with_properties.zip | Bin 0 -> 1532 bytes 2 files changed, 38 insertions(+) create mode 100644 tests/dataset_with_properties.zip diff --git a/tests/darwin/dataset/remote_dataset_test.py b/tests/darwin/dataset/remote_dataset_test.py index 7ae2625a9..3df57d8a5 100644 --- a/tests/darwin/dataset/remote_dataset_test.py +++ b/tests/darwin/dataset/remote_dataset_test.py @@ -766,6 +766,44 @@ def test_raises_if_release_format_is_not_json( with pytest.raises(UnsupportedExportFormat): remote_dataset.pull(release=a_release) + @patch("platform.system", return_value="Linux") + def test_moves_properties_metadata_file( + self, system_mock: MagicMock, remote_dataset: RemoteDataset + ): + stub_release_response = Release( + "dataset-slug", + "team-slug", + "0.1.0", + "release-name", + "http://darwin-fake-url.com", + datetime.now(), + None, + None, + True, + True, + "json", + ) + + def fake_download_zip(self, path): + zip: Path = Path("tests/dataset_with_properties.zip") + shutil.copy(zip, path) + return path + + with patch.object( + RemoteDataset, "get_release", return_value=stub_release_response + ) as get_release_stub: + with patch.object(Release, "download_zip", new=fake_download_zip): + remote_dataset.pull(only_annotations=True) + metadata_path = ( + remote_dataset.local_path + / "releases" + / "latest" + / "annotations" + / ".v7" + / "metadata.json" + ) + assert metadata_path.exists() + @pytest.fixture def dataset_item(dataset_slug: str) -> DatasetItem: diff --git a/tests/dataset_with_properties.zip b/tests/dataset_with_properties.zip new file mode 100644 index 0000000000000000000000000000000000000000..f2877fa988fd2e746a0d26b08730110b434cf467 GIT binary patch literal 1532 zcmWIWW@Zs#-~dAASmy`^D7ePTz#zt;z>t<&RFqhhshg7sW$9%V=jVln@G`KAsqIV6 z=$B6e;nE6j21b^zj0_AwvltkrhWPf+HW1kRJlsX5YSm8@&NW+J&A$=7)rr~cf$j~C z*%wNt{*Rk2kYskUtlIHg^0Twj`#0VmpEDP& zO{=?}z4!5jpy1_8YWZ!QdCzEnU%Pr4i|m!gRlXC-8J9@>YD(*wp0se*N@F*l74j;q z$2RtRNC|{I30P#ooBSkM=aU9=$S}xXd&dvjwf~z&Qe4n~6!+D$jv+uGyPwg;$ zde5M5(f`tfl0PfLwN^;1&p9~#g8b%7&owyqZw%|M?fWqEXNloTL9eV`^NS_DH&~TT zexO;X8@A%jrM~?Cl9Dsq654CI*J`fLdMw>nCbs6)!n2keo^i`M9^d2H{>AA{yXOnt zn`galRX+2&7O;DDX?pMy&XfC$Zg?NFe}1IHCL{Hh-Ai4+D(CAQ3YVDU`>lF|LoV%b zk(ey>G;!tv4PC>9iIajhm9Lq!JoWFcje?tJzbS4gUg+@h_`_Roiq01q7A)#FJbm>2 zR~4!Dq&>yC{cgRizZD`H)%1^Qzp%JDW5>Upu+_k`s3Q-g)Wne<9m|FB4xh7C-yF%knR4fHymb3!fXm4if_dDAfgk z(<3MuX8BKqr$-Q<1(+W7%FOk_sSlLuvazQ=hzXz+9q5ZNL5PikK@e&}ZfZ$lN@7VO zA~9EWLlQEU#JqQMcJ?6yk+%K6wY}_fFLbZXTTq|sVVT)KYoc1DUY7d5@2X#~buCzv zEaG)e^?mWZWjFhq6z(jz6!M`!t|iilPkH|3r&8G;R&F&6I)8Z!)7GUn)210jsZ~u{ z(7Szw?;O>D!`mNhxaz6-DmeM$N>=}DlYqv?E1b*S&mIx)n=q;I+6vjHhg%&tUjb+elUHfHa>-ISRUt#Ig5*Qd~ntHV}!yDqhy^>Ero@0B7x8vBFSa_o7MpMTNf z)F+#w;=OUKTU1-!W8AG?1+^{>XAR4_b4t_o`l^F-UmlyHAE>BrX!BG|ymn=aOX>^W zg!;(RJ^E9Z#@_9#emlol$Hd6oXaCawoaYXxZM!7P99Eb*^|lQ2^Fs~qyE1iKcLfIi zNS*uql&4N(<{Q!TvyQUO*jrfidfD9z-?mPBvFdnbjnUp`&RMByu&cz z_rj@Ja#g3&E}p#pK2Slm`p}f;PbS|K5Zc5gT6yY5yo?y*yLo>&v+rGd^H*-Qu&&@2x5^}z`^VQnSd>sg2X}Kt>Xk>vL-IkB3unhw#YsLC0iI+(s&7K ZB1wrjz?&5qGz<*PKqv|fsEaHh9ssyAV1NJs literal 0 HcmV?d00001