Merge pull request #40 from airware/some-udpates

Some udpates

.circleci/config.yml

@@ -60,7 +60,7 @@ references:
         - repo-v2-
 
   py27_requirements_cache_key: &py27_requirements_cache_key
-    py27-dependencies-v7-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+    py27-dependencies-v7-{{ checksum "requirements-ci.txt" }}-{{ checksum "setup.py" }}
 
   py27_requirements_backup_cache_key: &py27_requirements_backup_cache_key
     py27-dependencies-v7-
@@ -72,7 +72,7 @@ references:
         - *py27_requirements_backup_cache_key
 
   py34_requirements_cache_key: &py34_requirements_cache_key
-    py34-dependencies-v7-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+    py34-dependencies-v7-{{ checksum "requirements-ci.txt" }}-{{ checksum "setup.py" }}
 
   py34_requirements_backup_cache_key: &py34_requirements_backup_cache_key
     py34-dependencies-v7-
@@ -84,7 +84,7 @@ references:
         - *py34_requirements_backup_cache_key
 
   py36_requirements_cache_key: &py36_requirements_cache_key
-    py36-dependencies-v7-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+    py36-dependencies-v7-{{ checksum "requirements-ci.txt" }}-{{ checksum "setup.py" }}
 
   py36_requirements_backup_cache_key: &py36_requirements_backup_cache_key
     py36-dependencies-v7-
@@ -96,7 +96,7 @@ references:
         - *py36_requirements_backup_cache_key
 
   py37_requirements_cache_key: &py37_requirements_cache_key
-    py37-dependencies-v7-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+    py37-dependencies-v7-{{ checksum "requirements-ci.txt" }}-{{ checksum "setup.py" }}
 
   py37_requirements_backup_cache_key: &py37_requirements_backup_cache_key
     py37-dependencies-v7-
@@ -128,7 +128,7 @@ jobs:
             mkdir -p venvs/lint
             python -m venv ./venvs/lint
             . venvs/lint/bin/activate
-            pip install -r requirements-dev.txt
+            pip install -r requirements-lint.txt
       - run:
           name: run linter
           command: |
@@ -153,9 +153,7 @@ jobs:
             mkdir -p venvs/27
             virtualenv venvs/27
             . venvs/27/bin/activate
-            pip install gdal==2.4.0 # Fix gdal version, this should match the Dockerfile version
-            pip install -r requirements.txt
-            pip install -r requirements-dev.txt
+            pip install -r requirements-ci.txt
             if [ ! -f venvs/27/lib/python2.7/site-packages/osgeo/_gdal_array.so ]; then cp /usr/local/gdal/lib/python2.7/site-packages/osgeo/_gdal_array.so venvs/27/lib/python2.7/site-packages/osgeo; fi
       # run tests!
       - run:
@@ -164,7 +162,6 @@ jobs:
           # no coveralls for py27
           command: |
             . venvs/27/bin/activate
-            pip install .
             circleci tests glob **/test/*.py | circleci tests split --split-by=timings --timings-type=filename | xargs python -m pytest -x --junitxml=test-reports/pytest-report.xml
       - store_artifacts:
           path: test-reports
@@ -178,7 +175,7 @@ jobs:
 
   py34_test:
     <<: *py34_container_config
-    parallelism: 4
+    parallelism: 1
     steps:
       - *restore_repo
       - *py34_restore_dependencies
@@ -188,16 +185,13 @@ jobs:
             mkdir -p venvs/34
             python -m venv ./venvs/34
             . venvs/34/bin/activate
-            pip install gdal==2.4.0 # Fix gdal version, this should match the Dockerfile version
-            pip install -r requirements.txt
-            pip install -r requirements-dev.txt
-            pip install .
+            pip install -r requirements-ci.txt
       # run tests!
       - run:
           name: run tests
           command: |
             . venvs/34/bin/activate
-            circleci tests glob **/test/*.py | circleci tests split --split-by=timings --timings-type=filename | xargs python -m pytest -x --junitxml=test-reports/pytest-report.xml
+            python scripts/pytest_parallel.py -x --junitxml=test-reports/pytest-report.xml
       - store_artifacts:
           path: test-reports
           destination: test-reports
@@ -210,7 +204,7 @@ jobs:
 
   py36_test:
     <<: *py36_container_config
-    parallelism: 4
+    parallelism: 1
     steps:
       - *restore_repo
       - *py36_restore_dependencies
@@ -220,16 +214,13 @@ jobs:
             mkdir -p venvs/36
             python -m venv ./venvs/36
             . venvs/36/bin/activate
-            pip install gdal==2.4.0 # Fix gdal version, this should match the Dockerfile version
-            pip install -r requirements.txt
-            pip install -r requirements-dev.txt
-            pip install .
+            pip install -r requirements-ci.txt
       # run tests!
       - run:
           name: run tests
           command: |
             . venvs/36/bin/activate
-            circleci tests glob **/test/*.py | circleci tests split --split-by=timings --timings-type=filename | xargs python -m pytest -x --junitxml=test-reports/pytest-report.xml
+            python scripts/pytest_parallel.py -x --junitxml=test-reports/pytest-report.xml
       - store_artifacts:
           path: test-reports
           destination: test-reports
@@ -242,7 +233,7 @@ jobs:
 
   py37_test:
     <<: *py37_container_config
-    parallelism: 4
+    parallelism: 1
     steps:
       - *restore_repo
       - *py37_restore_dependencies
@@ -255,18 +246,24 @@ jobs:
             # Workaround from https://github.com/PyWavelets/pywt/issues/375
             pip install Cython
             pip install numpy>=1.13
-            pip install gdal==2.4.0 # Fix gdal version, this should match the Dockerfile version
-            pip install -r requirements.txt
-            pip install -r requirements-dev.txt
-            pip install .
+
+            pip install -r requirements-ci.txt
       # run tests!
       - run:
           name: run tests
           command: |
             . venvs/37/bin/activate
-            circleci tests glob **/test/*.py | circleci tests split --split-by=timings --timings-type=filename | xargs python -m pytest --cov=buzzard --cov-report=term --cov-report=xml --junitxml=test-reports/pytest-report.xml
+            python scripts/pytest_parallel.py -x --cov=buzzard --cov-report=term --cov-report=xml:test-reports/coverage.xml --junitxml=test-reports/pytest-report.xml
+            ls -l test-reports
+            # codecov first concatenate the coverage*.xml files and upload it
+            # it does not seem to care about the pytest-report*.xml, is it a problem?
+            # https://circleci.com/gh/airware/buzzard/1564#queue-placeholder/containers/0
+            # It used to upload those.
+            # https://circleci.com/gh/airware/buzzard/1506#artifacts/containers/0
             codecov
+
       - store_artifacts:
+          # We now store the coverage*.xml files, is it a problem?
           path: test-reports
           destination: test-reports
       - store_test_results:

.circleci/images/base-python/Dockerfile

@@ -32,7 +32,7 @@ ARG LIBSPATIALINDEX_VERSION=1.8.5
 #                                      #
 ########################################
 
-LABEL maintainer="Airware Corporation <devs@airware.com>"
+LABEL maintainer="Delair <nicolas.goguey@delair.aero>"
 
 ##############################################
 #                                            #

.coveragerc

@@ -1,2 +1,2 @@
 [run]
-omit = buzzard/test/*,buzzard/srs/*,buzzard/_tools/conv/*
+omit = buzzard/test/*,buzzard/srs/*,buzzard/_tools/conv/*,scripts/*,buzzard/algo/*

.gitignore

@@ -123,3 +123,4 @@ doc/*.shx
 doc/*.prj
 .pytest_cache
 dask-worker-space
+test-reports

AUTHORS.md

@@ -1,6 +1,7 @@
 # Authors
-- Nicolas Goguey <ngoguey@airware.com>
-- Hervé Nivon <hnivon@airware.com>
+- Nicolas Goguey <nicolas.goguey@delair.aero>
+- Hervé Nivon <herve.nivon@delair.aero>
 - Amaury Zarzelli <amaury.zarzelli@ign.fr>
+- Rubenach Théo <theo.rubenach@delair.aero>
 
 See also [Contributors](https://github.com/airware/buzzard/graphs/contributors).

CONTRIBUTING.md

@@ -4,7 +4,7 @@ First of all, we thank you for your interest in our project and any contribution
 
 ## Before you start
 
-`buzzard` is under active development. bugs, enhancements and new features suggestions are warm welcome. See [`TODO`](https://github.com/airware/buzzard/wiki/TODO) for a glimpse to our actual pipeline.
+`buzzard` is under active development. bugs, enhancements and new features suggestions are warm welcome. See [`TODO`](https://www.notion.so/buzzard/2c94ef6ee8da4d6280834129cc00f4d2?v=334ead18796342feb32ba85ccdfcf69f) for a glimpse to our actual pipeline.
 
 If you have discovered a new bug, please search for existing issues, open and closed, before creating a new one. You can track new ones creating a Github [issue](https://github.com/airware/buzzard/issues). `buzzard` execution environments can be quite complex. Please describe it as much as possible. Reproduction steps and dataset are more than useful!
 
@@ -20,8 +20,6 @@ Any contributions should pass all tests.
 2. Commit your changes to your fork
 3. Create a new pull request [(quick link)](https://github.com/airware/buzzard/compare)
 
-Additional consideration: at least for now, we aim at keeping `buzzard` compatible with both `python27` and `python3`. Please keep that in mind. 
-
 If you would like to make a significant change, please open an [issue on Github](https://github.com/airware/buzzard/issues) to discuss it before.
 
 ## Rights

README.md

@@ -6,7 +6,7 @@ In a nutshell, `buzzard` reads and writes geospatial raster and vector data.
   <img src="img/buzzard.png"><br><br>
 </div>
 
-[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/airware/asl-validator/blob/master/LICENSE)
+[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/airware/buzzard/blob/master/LICENSE)
 [![CircleCI](https://circleci.com/gh/airware/buzzard/tree/master.svg?style=shield&circle-token=9d41310f0eb3f8ff120a7103ba2d7ee5d5d628b7)](https://circleci.com/gh/airware/buzzard/tree/master)
 [![codecov](https://codecov.io/gh/airware/buzzard/branch/master/graph/badge.svg?token=FbWmLGplCq)](https://codecov.io/gh/airware/buzzard)
 
@@ -33,11 +33,11 @@ Table of Contents
 - a `gdal`/`ogr`/`osr` wrapper
 - designed to hide all cumbersome operations while working with GIS files
 - designed for data science workflows
-- under active development (see [`TODO`](https://github.com/airware/buzzard/wiki/TODO))
+- under active development (see [`todo`](https://www.notion.so/buzzard/2c94ef6ee8da4d6280834129cc00f4d2?v=334ead18796342feb32ba85ccdfcf69f))
 - tested with `pytest` with python 3.4 and python 3.7
 
 ## `buzzard` contains
-- a class to open/read/write/create GIS files: [`DataSource`](./buzzard/_datasource.py)
+- a class to open/read/write/create GIS files: [`Dataset`](./buzzard/_dataset.py)
 - a toolbox class designed to locate a rectangle in both image space and geometry space: [`Footprint`](./buzzard/_footprint.py)
 
 ## Simple example
@@ -51,7 +51,7 @@ import matplotlib.pyplot as plt
 rgb_path = 'path/to/raster.file'
 polygons_path = 'path/to/vector.file'
 
-ds = buzz.DataSource()
+ds = buzz.Dataset()
 ds.open_raster('rgb', rgb_path)
 ds.open_vector('polygons', polygons_path)
 
@@ -62,8 +62,8 @@ for poly in ds.polygons.iter_data(None):
     fp = ds.rgb.fp.intersection(poly)
 
     # Read rgb at `fp` to a numpy array
-    rgb = ds.rgb.get_data(band=(1, 2, 3), fp=fp).astype('uint8')
-    alpha = ds.rgb.get_data(band=4, fp=fp).astype('uint8')
+    rgb = ds.rgb.get_data(fp=fp, channels=(0, 1, 2)).astype('uint8')
+    alpha = ds.rgb.get_data(fp=fp, channels=3).astype('uint8')
 
     # Create a boolean mask as a numpy array from a shapely polygon
     mask = np.invert(fp.burn_polygons(poly))
@@ -102,7 +102,7 @@ The following table lists dependencies along with the minimum version, their sta
 
 | Library          | Version  | Mandatory | License                                                                              | Comment                                                       |
 |------------------|----------|-----------|--------------------------------------------------------------------------------------|---------------------------------------------------------------|
-| gdal             | >=2.1.3  | Yes       | [MIT/X](https://github.com/OSGeo/gdal/blob/trunk/gdal/LICENSE.TXT)                   | Hard to install. Will be included in `buzzard` wheels         |
+| gdal             | >=2.3.3  | Yes       | [MIT/X](https://github.com/OSGeo/gdal/blob/trunk/gdal/LICENSE.TXT)                   | Hard to install. Will be included in `buzzard` wheels         |
 | opencv-python    | >=3.1.0  | Yes       | [3-clause BSD](http://opencv.org/license.html)                                       | Easy to install with `opencv-python` wheels. Will be optional |
 | shapely          | >=1.6.1  | Yes       | [3-clause BSD](https://github.com/Toblerity/Shapely/blob/master/LICENSE.txt)         |                                                               |
 | affine           | >=2.0.0  | Yes       | [3-clause BSD](https://github.com/sgillies/affine/blob/master/LICENSE.txt)           |                                                               |
@@ -117,33 +117,58 @@ The following table lists dependencies along with the minimum version, their sta
 | pytest           | >=3.2.2  | No        | [MIT](https://docs.pytest.org/en/latest/license.html)                                | Only for tests                                                |
 | attrdict         | >=2.0.0  | No        | [MIT](https://github.com/bcj/AttrDict/blob/master/LICENSE.txt)                       | Only for tests                                                |
 
-## How to install
-
-### Package manager and pip
+## How to install from terminal
+### Anaconda and pip
 ```sh
-# Install GDAL
-# Windows: http://www.lfd.uci.edu/~gohlke/pythonlibs/#gdal
-# MacOS: brew install gdal && brew tap osgeo/osgeo4mac && brew tap --repair && brew install gdal2 && export PATH="/usr/local/opt/gdal2/bin:$PATH" && pip install 'gdal==2.1.3'
-# Ubuntu: apt-get install python-gdal=2.1.3+dfsg-1~xenial2 libproj-dev libgdal-dev gdal-bin
+# Step 1 - Install Anaconda
+# https://www.anaconda.com/download/
 
-# Install buzzard
+# Step 2 - Create env
+conda create -n buzz python gdal>=2.3.3 shapely rtree -c 'conda-forge'
+
+# Step 3 - Activate env
+conda activate buzz
+
+# Step 4 - Install buzzard
 pip install buzzard
 ```
 
-### Anaconda and pip
+### Docker
 ```sh
-# Install Anaconda
-# https://www.anaconda.com/download/
-
-# Create env
-conda create -n buzz python=3.6 gdal opencv scipy shapely -c 'conda-forge'
+docker build -t buzz --build-arg PYTHON_VERSION=3.7 https://raw.githubusercontent.com/airware/buzzard/master/.circleci/images/base-python/Dockerfile
+docker run -it --rm buzz bash
+pip install buzzard
 
-# Activate env
-# Windows: activate buzz
-# Linux, MacOS: source activate buzz
+```
 
-# Install buzzard
-pip install buzzard
+### Package manager and pip
+```sh
+# Step 1 - Install GDAL and rtree
+# Windows:
+# https://www.lfd.uci.edu/~gohlke/pythonlibs/#gdal
+# https://www.lfd.uci.edu/~gohlke/pythonlibs/#rtree
+
+# MacOS:
+brew install gdal
+brew tap osgeo/osgeo4mac
+brew tap --repair
+brew install gdal2
+brew install spatialindex
+export PATH="/usr/local/opt/gdal2/bin:$PATH"
+python -m pip install 'gdal==2.3.3'
+
+# Ubuntu:
+sudo add-apt-repository ppa:ubuntugis/ppa
+sudo apt-get update
+sudo apt-get install gdal-bin
+sudo apt-get install libgdal-dev
+sudo apt-get install python3-rtree
+export CPLUS_INCLUDE_PATH=/usr/include/gdal
+export C_INCLUDE_PATH=/usr/include/gdal
+pip install GDAL
+
+# Step 2 - Install buzzard
+python -m pip install buzzard
 ```
 
 ## Supported Python versions
@@ -183,6 +208,6 @@ See [LICENSE](./LICENSE) and [NOTICE](./NOTICE).
 - [examples](./doc/examples.ipynb)
 - [classes](https://github.com/airware/buzzard/wiki/Classes)
 - [wiki](https://github.com/airware/buzzard/wiki)
-- [todo](https://github.com/airware/buzzard/wiki/TODO)
+- [todo](https://www.notion.so/buzzard/2c94ef6ee8da4d6280834129cc00f4d2?v=334ead18796342feb32ba85ccdfcf69f)
 
 ------------------------------------------------------------------------------------------------------------------------