add test for dataset with start_datetime and end_datetime properties (#…

…165) * Add missing vcr cassette for 3 new datetime tests * fixing formatting on tests * cleaning up code and rerecording cassettes * updating name of cassette * Fixes to test_2_datetime_filters_to_start_and_end_datetime_fields. * Formatting, typos. * removing MonkeyPatch test * Improvements to test_2_datetime_filters_to_start_and_end_datetime_fields. * Remove unnecessary vcr.py usage. * Linting * Improve test_1_datetime_filter_to_start_and_end_datetimes. * Changelog Co-authored-by: Alex G Rice <alex@radiant.earth>
radiantearth · Oct 11, 2022 · 840be92 · 840be92
1 parent 501aa7c
commit 840be92
Show file tree

Hide file tree

Showing 20 changed files with 691 additions and 506 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added an optional asset_output_dir parameter so that the assets can be downloaded into a separate directory from the STAC Catalog if the user specifies and selects a separate asset output directory. ([149](https://github.com/radiantearth/radiant-mlhub/pull/149))
 
+- Add datetime filter test to account for datasets that have start_datetime and end_datetime properties. ([165](https://github.com/radiantearth/radiant-mlhub/pull/165))
+
 ### Removed
 
 ### Changed

diff --git a/radiant_mlhub/client/catalog_downloader.py b/radiant_mlhub/client/catalog_downloader.py
@@ -607,8 +607,8 @@ def _download_asset_worker(
                     asset_rec = AssetRecord(
                         asset_save_path=asset_save_path,
                         asset_url=asset_url,
-                        collection_id=collection_id,  # TODO yank?
-                        item_id=item_id,  # TODO: yank?
+                        collection_id=collection_id,
+                        item_id=item_id,
                         asset_key=asset_key,
                     )
                     asset_list.append(asset_rec)

diff --git a/test/data/datasets/su_sar_moisture_content_main.json b/test/data/datasets/su_sar_moisture_content_main.json
@@ -0,0 +1,71 @@
+{
+  "id": "su_sar_moisture_content_main",
+  "bbox": {
+    "coordinates": [
+      [
+        [
+          [
+            -123.5313889,
+            28.3
+          ],
+          [
+            -123.5313889,
+            48.41361111
+          ],
+          [
+            -93.82277778,
+            48.41361111
+          ],
+          [
+            -93.82277778,
+            28.3
+          ],
+          [
+            -123.5313889,
+            28.3
+          ]
+        ]
+      ]
+    ],
+    "type": "MultiPolygon"
+  },
+  "citation": "Rao, K., Williams, A.P., Fortin, J. & Konings, A.G. (2020). SAR-enhanced mapping of live fuel moisture content. Remote Sens. Environ., 245.",
+  "collections": [
+    {
+      "id": "su_sar_moisture_content",
+      "types": [
+        "labels"
+      ],
+      "license": "[CC BY-NC-ND 4.0](https://creativecommons.org/licenses/by-nc-nd/4.0/)"
+    }
+  ],
+  "creator_contact": {
+    "contact": "kkrao.j@gmail.com",
+    "creator": "[Stanford University](https://www.stanford.edu/)"
+  },
+  "date_added": "2020-08-13T00:00:00+00:00",
+  "date_modified": "2020-08-13T00:00:00+00:00",
+  "description": "This data contains manually collected live fuel moisture measurements in the western United States and remotely-sensed variables. Live fuel moisture represents the mass of water in live vegetation elements like leaves, needles, and twigs divided by its oven-dried mass. It is represented in percentages. Higher the live fuel moisture, wetter the vegetation elements, and vice versa. Live fuel moisture measurements were collected by the United States Forest Service and are available from the [National Fuel Moisture Database](https://www.wfas.net/index.php/national-fuel-moisture-database-moisture-drought-103). Each row of the data corresponds to one unique ground measurement of live fuel moisture (column named \"percent(t)\") matched with various remotely-sensed observables that may be used to predict live fuel moisture. The live fuel moisture is sampled for representative species within a 5-acre plot (or 20,000 m2) centered at the location described by the columns \"latitude\" and \"longitude\" on the day described by the column \"date\". All other columns represent remotely-sensed observables from satellites (e.g., Sentinel-1 and Landsat-8) or maps (e.g., soil texture). Temporally varying remotely-sensed observables are interpolated to 15-day periods and are provided for the date closest to the day of ground-measurement as well as for 6 fortnights preceding the day of live fuel moisture measurement. The time series of satellite data may allow for greater predictability of live fuel moisture.",
+  "documentation_link": "https://radiantearth.blob.core.windows.net/mlhub/su-sar-moisture-content/documentation.pdf",
+  "doi": "10.1016/j.rse.2020.111797",
+  "publications": [
+    {
+      "url": "https://dx.doi.org/10.1016/j.rse.2020.111797",
+      "title": "SAR-enhanced mapping of live fuel moisture content",
+      "author_name": "Krishna Rao",
+      "author_url": "https://www.linkedin.com/in/kkraoj/"
+    }
+  ],
+  "registry": "https://mlhub.earth/data/su_sar_moisture_content_main",
+  "status": "ready",
+  "tags": [
+    "sentinel-1",
+    "landsat 8",
+    "sar",
+    "live fuel moisture",
+    "wildfire"
+  ],
+  "title": "Western USA Live Fuel Moisture",
+  "tools_applications": null,
+  "tutorials": null
+}
diff --git a/test/data/datasets/su_sar_moisture_content_main.tar.gz b/test/data/datasets/su_sar_moisture_content_main.tar.gz
diff --git a/...ownload_with_2_datetime_filter_works.yaml → ...time_filter_to_single_datetime_field.yaml b/...ownload_with_2_datetime_filter_works.yaml → ...time_filter_to_single_datetime_field.yaml
@@ -9,14 +9,14 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
     method: GET
     uri: https://staging.api.radiant.earth/mlhub/v1/datasets/nasa_marine_debris
   response:
     body:
       string: !!binary |
-        H4sIAAGGH2MC/61W72/bNhD9Vwh9GBpMlvwzif1pabK1AeK2WIINQ1IYlHiWuFKiRlJ23aL/+x4p
+        H4sIAK6KLGMC/61W72/bNhD9Vwh9GBpMlvwzif1pabK1AeK2WIINQ1IYlHiWuFKiRlJ23aL/+x4p
         OXa6pvuwGQFik7y7x3fv7vg5kiJaRDW3fFVxI2taCcqMtFEcZZn+GC0+R7nWRsiaO7LR4h6fyTSZ
         TYbD8Xw8P4tnyfT0fHI6OZu9j5/uTObJaHh+NhtNZnPsDc7PEyyPRvPRdPT9zed9Hu3gE0du1xDw
         L1vl5DutdoWuoy9xlEvHncT3RXRb8jJmF0nM7kpdcRuzG3z/gS25/UC7mC0T9mI8HI9O2EO0DAyw
@@ -48,11 +48,11 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 12 Sep 2022 19:18:24 GMT
+      - Thu, 22 Sep 2022 16:17:50 GMT
       Vary:
       - Accept-Encoding
       X-Azure-Ref:
-      - 0AYYfYwAAAAB3KjELeduHQI7q7XI0pNKqREVOMzAxMDAwMTA5MDM5ADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
+      - 0roosYwAAAAALmzz4onDORoLFa4h1G0GcQVRMMzMxMDAwMTA5MDQ5ADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
       X-Cache:
       - CONFIG_NOCACHE
     status:
@@ -68,8 +68,8 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
       x-ms-version:
       - '2021-06-08'
     method: GET
@@ -81,11 +81,11 @@ interactions:
       Content-Length:
       - '0'
       Date:
-      - Mon, 12 Sep 2022 19:18:25 GMT
+      - Thu, 22 Sep 2022 16:17:50 GMT
       Location:
       - https://radiantearth.blob.core.windows.net/mlhub/catalogs/nasa_marine_debris.tar.gz
       X-Azure-Ref:
-      - 0AoYfYwAAAAD6wH+qjdvPQKZdUtFa+Nd8REVOMzAxMDAwMTA5MDUxADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
+      - 0r4osYwAAAADW5CQ/FAtGSYIfujtD4uU1QVRBRURHRTEyMjEAMDg0MTE2ZjAtMmYwMy00OTdiLTllMjEtNzNlNjQ1NGMwNjcw
       X-Cache:
       - CONFIG_NOCACHE
     status:
@@ -101,8 +101,8 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
       x-ms-version:
       - '2021-06-08'
     method: GET
@@ -4747,7 +4747,7 @@ interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Mon, 12 Sep 2022 19:18:27 GMT
+      - Thu, 22 Sep 2022 16:17:51 GMT
       ETag:
       - '"0x8DA27B5CF6B14FA"'
       Last-Modified:
@@ -4763,7 +4763,7 @@ interactions:
       x-ms-lease-status:
       - unlocked
       x-ms-request-id:
-      - 6602dd19-e01e-0038-3edc-c6f171000000
+      - 3a84178e-b01e-0061-069e-cee151000000
       x-ms-server-encrypted:
       - 'true'
       x-ms-version:

diff --git a/...ownload_with_1_datetime_filter_works.yaml → ...ime_filters_to_single_datetime_field.yaml b/...ownload_with_1_datetime_filter_works.yaml → ...ime_filters_to_single_datetime_field.yaml
@@ -9,14 +9,14 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
     method: GET
     uri: https://staging.api.radiant.earth/mlhub/v1/datasets/nasa_marine_debris
   response:
     body:
       string: !!binary |
-        H4sIAP2FH2MC/61W72/bNhD9Vwh9GBpMlvwzif1pabK1AeK2WIINQ1IYlHiWuFKiRlJ23aL/+x4p
+        H4sIALaKLGMC/61W72/bNhD9Vwh9GBpMlvwzif1pabK1AeK2WIINQ1IYlHiWuFKiRlJ23aL/+x4p
         OXa6pvuwGQFik7y7x3fv7vg5kiJaRDW3fFVxI2taCcqMtFEcZZn+GC0+R7nWRsiaO7LR4h6fyTSZ
         TYbD8Xw8P4tnyfT0fHI6OZu9j5/uTObJaHh+NhtNZnPsDc7PEyyPRvPRdPT9zed9Hu3gE0du1xDw
         L1vl5DutdoWuoy9xlEvHncT3RXRb8jJmF0nM7kpdcRuzG3z/gS25/UC7mC0T9mI8HI9O2EO0DAyw
@@ -48,11 +48,11 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Mon, 12 Sep 2022 19:18:21 GMT
+      - Thu, 22 Sep 2022 16:17:58 GMT
       Vary:
       - Accept-Encoding
       X-Azure-Ref:
-      - 0/YUfYwAAAAB6naq/lEGLSq/uyi+bu11iREVOMzAxMDAwMTA5MDI5ADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
+      - 0toosYwAAAAAoJpYxV0X7TIkf/J4Wxf58QVRMMzMxMDAwMTEwMDExADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
       X-Cache:
       - CONFIG_NOCACHE
     status:
@@ -68,8 +68,8 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
       x-ms-version:
       - '2021-06-08'
     method: GET
@@ -81,11 +81,11 @@ interactions:
       Content-Length:
       - '0'
       Date:
-      - Mon, 12 Sep 2022 19:18:22 GMT
+      - Thu, 22 Sep 2022 16:17:58 GMT
       Location:
       - https://radiantearth.blob.core.windows.net/mlhub/catalogs/nasa_marine_debris.tar.gz
       X-Azure-Ref:
-      - 0/oUfYwAAAABbbs7hqx4JRo5GeQ5Mp2LoREVOMzAxMDAwMTA5MDA5ADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
+      - 0toosYwAAAAANJPj/3sK9Qb/vmIHnJdOWQVRMMzMxMDAwMTA5MDQ1ADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
       X-Cache:
       - CONFIG_NOCACHE
     status:
@@ -101,8 +101,8 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
       x-ms-version:
       - '2021-06-08'
     method: GET
@@ -4747,7 +4747,7 @@ interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Mon, 12 Sep 2022 19:18:22 GMT
+      - Thu, 22 Sep 2022 16:17:58 GMT
       ETag:
       - '"0x8DA27B5CF6B14FA"'
       Last-Modified:
@@ -4763,7 +4763,7 @@ interactions:
       x-ms-lease-status:
       - unlocked
       x-ms-request-id:
-      - 92bf6ca9-e01e-006c-18dc-c63e4a000000
+      - fde964fe-301e-0011-3e9e-ce1351000000
       x-ms-server-encrypted:
       - 'true'
       x-ms-version:

diff --git a/test/models/cassettes/test_dataset/TestDataset.test_asset_output_dir_works.yaml b/test/models/cassettes/test_dataset/TestDataset.test_asset_output_dir_works.yaml
@@ -9,14 +9,14 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
     method: GET
     uri: https://staging.api.radiant.earth/mlhub/v1/datasets/nasa_marine_debris
   response:
     body:
       string: !!binary |
-        H4sIAIpqGmMC/61W72/bNhD9Vwh9GBpMlvwzif1pabK1AeK2WIINQ1IYlHiWuFKiRlJ23aL/+x4p
+        H4sIAKqKLGMC/61W72/bNhD9Vwh9GBpMlvwzif1pabK1AeK2WIINQ1IYlHiWuFKiRlJ23aL/+x4p
         OXa6pvuwGQFik7y7x3fv7vg5kiJaRDW3fFVxI2taCcqMtFEcZZn+GC0+R7nWRsiaO7LR4h6fyTSZ
         TYbD8Xw8P4tnyfT0fHI6OZu9j5/uTObJaHh+NhtNZnPsDc7PEyyPRvPRdPT9zed9Hu3gE0du1xDw
         L1vl5DutdoWuoy9xlEvHncT3RXRb8jJmF0nM7kpdcRuzG3z/gS25/UC7mC0T9mI8HI9O2EO0DAyw
@@ -48,11 +48,11 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 08 Sep 2022 22:19:54 GMT
+      - Thu, 22 Sep 2022 16:17:45 GMT
       Vary:
       - Accept-Encoding
       X-Azure-Ref:
-      - 0imoaYwAAAACkAabMo1OyQ4W+lqZv21lPREVOMzAxMDAwMTEwMDQ1ADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
+      - 0qoosYwAAAAAExTBHPhJ8QY6RkkwsgIJDQVRBRURHRTEyMTYAMDg0MTE2ZjAtMmYwMy00OTdiLTllMjEtNzNlNjQ1NGMwNjcw
       X-Cache:
       - CONFIG_NOCACHE
     status:
@@ -68,8 +68,8 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
       x-ms-version:
       - '2021-06-08'
     method: GET
@@ -81,11 +81,11 @@ interactions:
       Content-Length:
       - '0'
       Date:
-      - Thu, 08 Sep 2022 22:19:55 GMT
+      - Thu, 22 Sep 2022 16:17:46 GMT
       Location:
       - https://radiantearth.blob.core.windows.net/mlhub/catalogs/nasa_marine_debris.tar.gz
       X-Azure-Ref:
-      - 0i2oaYwAAAAB+Qmkrz21pTJWCRH9RHjDbREVOMzAxMDAwMTEwMDUzADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
+      - 0qoosYwAAAACd2FYeaoi7T6R7j+6sU17wQVRMMzMxMDAwMTA5MDExADA4NDExNmYwLTJmMDMtNDk3Yi05ZTIxLTczZTY0NTRjMDY3MA==
       X-Cache:
       - CONFIG_NOCACHE
     status:
@@ -101,8 +101,8 @@ interactions:
       Connection:
       - keep-alive
       User-Agent:
-      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.6.0: Wed Aug 10 14:25:27 PDT
-        2022; root:xnu-8020.141.5~2/RELEASE_X86_64)'
+      - 'radiant_mlhub/0.5.2 (Darwin Kernel Version 21.3.0: Wed Jan  5 21:37:58 PST
+        2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000)'
       x-ms-version:
       - '2021-06-08'
     method: GET
@@ -4747,7 +4747,7 @@ interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Thu, 08 Sep 2022 22:19:56 GMT
+      - Thu, 22 Sep 2022 16:17:46 GMT
       ETag:
       - '"0x8DA27B5CF6B14FA"'
       Last-Modified:
@@ -4763,7 +4763,7 @@ interactions:
       x-ms-lease-status:
       - unlocked
       x-ms-request-id:
-      - 1b76cbc0-601e-0030-73d1-c3a960000000
+      - 6375cc7a-c01e-0016-319e-cec25f000000
       x-ms-server-encrypted:
       - 'true'
       x-ms-version: