Merge pull request #91 from sertit/0.20.0

0.20.0

.github/workflows/publish.yml

@@ -15,10 +15,10 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Check that the current version isn't already on PyPi
         run: |

.gitlab-ci.yml

@@ -11,7 +11,7 @@ variables:
   EOREADER_CHANGES: eoreader/**/[^_]*.{py,xml}
 
 lint:
-  image: python:3.8-buster
+  image: python:3.9-buster
   stage: lint
   script:
     - python -m pip install --upgrade pip
@@ -26,16 +26,17 @@ lint:
 
 # Test with data on local disk
 pytest:
-  image: $EEO_DEPS:38
+  image: $EEO_DEPS:latest
   stage: test
   variables:
     CI_EOREADER_USE_S3: "0"
   before_script:
     - python -m pip install --upgrade pip
     - pip install --ignore-installed PyYAML
+    - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_on_disk.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_on_disk.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_on_disk.html
@@ -59,16 +60,17 @@ pytest:
 # Test with data on S3
 # TODO: create S3-specific CI ?
 pytest_s3:
-  image: $EEO_DEPS:38
+  image: $EEO_DEPS:latest
   stage: test
   variables:
     EOREADER_USE_DASK: "0"
   before_script:
     - python -m pip install --upgrade pip
     - pip install --ignore-installed PyYAML
+    - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_s3.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_s3.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_s3.html
@@ -91,41 +93,17 @@ pytest_s3:
 
 # Test SNAP with SNAP
 pytest_end_to_end:
-  image: $EEO_DEPS:38
-  stage: test
-  variables:
-    EOREADER_USE_DASK: "0"
-  before_script:
-    - python -m pip install --upgrade pip
-    - pip install --ignore-installed PyYAML
-    - pip install -e .
-  script:
-    - pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
-  artifacts:
-    paths:
-      - ${CI_PROJECT_DIR}/cov_e2e.html
-  coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
-  tags:
-    - sertit
-    - linux
-    - high_memory
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "END_TO_END"'
-      when: always
-    - if: $CI_COMMIT_TAG
-      when: always
-
-pytest_end_to_end39:
   image: $EEO_DEPS:latest
   stage: test
   variables:
     EOREADER_USE_DASK: "0"
   before_script:
     - python -m pip install --upgrade pip
     - pip install --ignore-installed PyYAML
+    - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html
@@ -139,7 +117,6 @@ pytest_end_to_end39:
       when: always
     - if: $CI_COMMIT_TAG
       when: always
-  needs: [ "pytest_end_to_end" ]
 
 pytest_end_to_end310:
   image: $EEO_DEPS:310
@@ -150,9 +127,10 @@ pytest_end_to_end310:
   before_script:
     - python -m pip install --upgrade pip
     - pip install --ignore-installed PyYAML
+    - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html
@@ -166,7 +144,7 @@ pytest_end_to_end310:
       when: always
     - if: $CI_COMMIT_TAG
       when: always
-  needs: [ "pytest_end_to_end39" ]
+  needs: [ "pytest_end_to_end" ]
 
 pytest_end_to_end311:
   image: $EEO_DEPS:311
@@ -177,9 +155,10 @@ pytest_end_to_end311:
   before_script:
     - python -m pip install --upgrade pip
     - pip install --ignore-installed PyYAML
+    - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html
@@ -195,21 +174,6 @@ pytest_end_to_end311:
       when: always
   needs: [ "pytest_end_to_end310" ]
 
-tox-linux-3.8:on-schedule:
-  image: $EO_CONTAINER:3.8
-  stage: weekly_tests
-  before_script:
-    - python -m pip install --upgrade pip
-    - pip install tox
-  script:
-    - tox -c tox.ini -e py38
-  tags:
-    - sertit
-    - linux
-    - high_memory
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "WEEKLY"'
-
 tox-linux-3.9:on-schedule:
   image: $EO_CONTAINER:3.9
   stage: weekly_tests
@@ -224,7 +188,6 @@ tox-linux-3.9:on-schedule:
     - high_memory
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "WEEKLY"'
-  needs: [ "tox-linux-3.8:on-schedule" ]
 
 tox-windows:on-schedule:
   stage: weekly_tests

CHANGES.md

@@ -1,5 +1,30 @@
 # Release History
 
+## 0.20.0 (2023-MM-DD)
+
+### Breaking Changes
+
+- **BREAKING CHANGES: Switching from `resolution` to `pixel_size` to avoid confusion about the definitions (especially for SAR data)** ([#82](https://github.com/sertit/eoreader/issues/82))
+- **BREAKING CHANGES: `load` function now returns a `xarray.Dataset`** ([#88](https://github.com/sertit/eoreader/issues/88))
+
+### Bug Fixes
+
+- FIX: Collocate bands before trying to create spectral indices: resolve the case where their size mismatches (i.e. in case of window or change of native pixel size)
+- FIX: Landsat band masking when specifying a custom resolution and a custom window
+- FIX: Round the default pixel_size of custom stacks
+- FIX: Convert some fields of STAC items from non JSON serializable dtypes to correct ones
+- FIX: Fix erroneous property set to `_get_raw_crs` for Maxar products
+
+### Other
+
+- ENH: Don't load data into memory when computing indices, ensuring tasks are delayed a bit longer ([#58](https://github.com/sertit/eoreader/issues/58))
+- DOCS: Add documentation about default CRS ([#87](https://github.com/sertit/eoreader/issues/87))
+- DEPS: Dropping support of Python 3.8 ([#81](https://github.com/sertit/eoreader/issues/81))
+- DEPS: Pin sertit to 1.27.0
+- INTERNAL: Better management of logs for deprecation warnings
+- INTERNAL: Refactoring `simplify_footprint` in `sertit` library
+- CI: Test that STAC items are serializable when added to a catalog
+
 ## 0.19.4 (2023-04-12)
 
 ### Bug Fixes
@@ -17,6 +42,7 @@
 
 ### Bug Fixes
 
+- OPTIM: Don't recompute stacks if already existing on disk
 - FIX: Fixing `Custom Stacks` when specifying `datetime=None` on creation
 - FIX: Fix regression for multi-swath DGM CSK data (huge region) ([#78](https://github.com/sertit/eoreader/issues/78))
 - FIX: Fix calibration issues with CSK HR data (using fallback GPT graph by default)
@@ -818,7 +844,7 @@
 
 ### Other
 
-- CI: Fixing `test_dems_https` and resetting DEM afterwards
+- CI: Fixing `test_dems_https` and resetting DEM afterward
 - CI: Fixing DEM and ds2 database management
 - DOC: Adding a FAQ page and enhancing the Main Features page (#3)
 - DOC: Read Metadata has its own paragraph in Main Features

CI/SCRIPTS/test_broken_s2.py

@@ -17,7 +17,7 @@
 @dask_env
 def test_broken_s2():
     """Function testing the support of broken Sentinel-2 constellation"""
-    res = 10.0 * 100
+    pixel_size = 10.0 * 100
 
     # ----------- Broken MTD -----------
     broken_mtd = broken_s2_path().joinpath(
@@ -30,8 +30,8 @@ def test_broken_s2():
     LOGGER.info(broken_mtd_prod)
     LOGGER.info(broken_mtd_prod.bands)
 
-    broken_mtd_prod.load(RED, resolution=res, clean_optical="clean")
-    broken_mtd_prod.load(NIR, resolution=res, clean_optical="nodata")
+    broken_mtd_prod.load(RED, pixel_size=pixel_size, clean_optical="clean")
+    broken_mtd_prod.load(NIR, pixel_size=pixel_size, clean_optical="nodata")
 
     # Invalid tests
     with pytest.raises(InvalidProductError):
@@ -52,20 +52,20 @@ def test_broken_s2():
     broken_detfoo_prod.footprint()
 
     broken_detfoo_prod.load(
-        RED, resolution=res, clean_optical="clean"
+        RED, pixel_size=pixel_size, clean_optical="clean"
     )  # Not corrupted band
 
     # Invalid tests
     # WARNING: This doesn't fail anymore!
     # with pytest.raises(InvalidProductError):
     #     broken_detfoo_prod.load(
-    #         NIR, resolution=res, clean_optical="nodata"
+    #         NIR, pixel_size=pixel_size, clean_optical="nodata"
     #     )  # Corrupted band
 
     # ----------- Broken MSK -----------
     broken_msk = broken_s2_path().joinpath(
         "S2B_MSIL2A_20220201T104149_N0400_R008_T31UFP_20220201T122857.SAFE"
     )
     broken_msk_prod = READER.open(broken_msk)
-    broken_msk_prod.load(RED, resolution=res, clean_optical="clean")
-    broken_mtd_prod.load(NIR, resolution=res, clean_optical="nodata")
+    broken_msk_prod.load(RED, pixel_size=pixel_size, clean_optical="clean")
+    broken_mtd_prod.load(NIR, pixel_size=pixel_size, clean_optical="nodata")

CI/SCRIPTS/test_custom.py

@@ -3,6 +3,7 @@
 
 import numpy as np
 import pytest
+from rasterio.windows import Window
 from sertit import ci
 
 from eoreader import EOREADER_NAME
@@ -46,7 +47,7 @@ def test_custom_optical():
         sensor_type=SensorType.OPTICAL,
         constellation="WV02",
         instrument="WW110",
-        resolution=2.0,
+        pixel_size=2.0,
         product_type="Ortho",
         band_map={BLUE: 1, GREEN: 2, RED: 3, NIR: 4, SWIR_1: 5},
     )
@@ -78,7 +79,10 @@ def test_custom_optical():
     extent_min = prod_min.extent()
     footprint_min = prod_min.footprint()
     crs_min = prod_min.crs()
-    bands = prod_min.load([BLUE, NIR, NDVI])
+    bands = prod_min.load(
+        [BLUE, NIR, NDVI],
+        window=Window(col_off=0, row_off=0, width=100, height=100),
+    )
 
     # Check attributes
     assert NDVI in bands
@@ -118,7 +122,10 @@ def test_custom_optical():
     extent_some = prod_some.extent()
     footprint_some = prod_some.footprint()
     crs_some = prod_some.crs()
-    bands = prod_some.load([HILLSHADE], resolution=200.0)
+    bands = prod_some.load(
+        [HILLSHADE],
+        window=Window(col_off=0, row_off=0, width=100, height=100),
+    )
 
     # Check attributes
     assert bands[HILLSHADE].attrs["long_name"] == "HILLSHADE"
@@ -160,15 +167,15 @@ def test_custom_sar():
         datetime="20210827T162210",
         constellation="ICEYE",
         instrument="SAR X-band",
-        resolution=6.0,
+        pixel_size=6.0,
         product_type="GRD",
         band_map={VV: 1, VV_DSPK: 2},
     )
     LOGGER.info(prod_sar)
     extent_sar = prod_sar.extent()
     footprint_sar = prod_sar.footprint()
     crs_sar = prod_sar.crs()
-    stack_sar = prod_sar.stack([VV, VV_DSPK], prod_sar.resolution * 10)
+    stack_sar = prod_sar.stack([VV, VV_DSPK], prod_sar.pixel_size * 10)
 
     # Errors
     with pytest.raises(AssertionError):
@@ -195,13 +202,13 @@ def test_custom_sar():
         product_type=None,
         instrument=None,
         datetime=None,
-        resolution=6.0,
+        pixel_size=6.0,
     )
     LOGGER.info(prod_wtf)
     extent_wtf = prod_wtf.extent()
     footprint_wtf = prod_wtf.footprint()
     crs_wtf = prod_wtf.crs()
-    stack_wtf = prod_wtf.stack([HH, RH], prod_wtf.resolution * 10)
+    stack_wtf = prod_wtf.stack([HH, RH], prod_wtf.pixel_size * 10)
 
     ci.assert_geom_equal(extent_sar, extent_wtf)
     ci.assert_geom_equal(footprint_sar, footprint_wtf)
@@ -223,7 +230,7 @@ def test_custom_wgs84():
         name="SPOT6_WGS84",
         datetime="20181218T090308",
         constellation="SPOT6",
-        resolution=1.5 * 15,
+        pixel_size=1.5 * 15,
         instrument="NAOMI",
         product_type="ORT",
         band_map={RED: 1, GREEN: 2, BLUE: 3, NIR: 4},
@@ -247,7 +254,7 @@ def test_custom_wgs84():
     assert root.findtext("datetime") == "2018-12-18T09:03:08"
     assert root.findtext("sensor_type") == "Optical"
     assert root.findtext("constellation") == "Spot-6"
-    assert root.findtext("resolution") == str(1.5 * 15)
+    assert root.findtext("pixel_size") == str(1.5 * 15)
     assert root.findtext("product_type") == "ORT"
     assert root.findtext("band_map") == "{'BLUE': 3, 'GREEN': 2, 'RED': 1, 'NIR': 4}"
     assert root.findtext("sun_azimuth") == "None"
@@ -268,7 +275,7 @@ def test_custom_wgs84():
         prod_wgs84.load(BLUE, size=[3863, 1049])[BLUE]  # noqa
 
     # Try non-available clouds and bands
-    assert prod_wgs84.load([]) == {}
+    assert len(prod_wgs84.load([])) == 0
     assert not prod_wgs84.has_bands(CLOUDS)
     with pytest.raises(AssertionError):
         prod_wgs84.load(CLOUDS, YELLOW)