Merge 2da7f1d into 132e646

.github/workflows/ci.yaml

@@ -0,0 +1,159 @@
+name: CI
+# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
+# https://docs.github.com/en/developers/webhooks-and-events/events/github-event-types#pullrequestevent
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.type }}
+  cancel-in-progress: true
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    name: lint and style checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 flake8-docstrings flake8-debugger flake8-bugbear pytest
+      - name: Run linting
+        run: |
+          flake8 uwsift/
+
+  website:
+    name: build website
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: "bash -l {0}"
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          environment-file: continuous_integration/environment.yaml
+          activate-environment: test-environment
+
+      - name: Install SIFT
+        shell: bash -l {0}
+        run: |
+          pip install sphinx sphinx_rtd_theme sphinxcontrib-apidoc sphinxcontrib-seqdiag sphinxcontrib-blockdiag blockdiag
+          pip install --no-deps -e .
+
+      - name: Run Sphinx Build
+        shell: bash -l {0}
+        run: |
+          cd doc
+          make html SPHINXOPTS="-W"
+
+  test:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: "bash -l {0}"
+    continue-on-error: ${{ matrix.experimental }}
+    needs: [lint]
+    strategy:
+      fail-fast: true
+      matrix:
+        # XXX: We don't currently have OpenGL installation on other platforms
+        #os: ["windows-latest", "ubuntu-latest", "macos-latest"]
+        os: ["ubuntu-latest"]
+        python-version: ["3.7", "3.9"]
+        experimental: [false]
+        include:
+          - python-version: "3.9"
+            os: "ubuntu-latest"
+            experimental: true
+
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      OS: ${{ matrix.os }}
+      UNSTABLE: ${{ matrix.experimental }}
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Prepare System Environment
+        run: |
+          # opengl system libraries
+          sudo apt-get update
+          cat continuous_integration/linux_full_deps_apt.txt | xargs sudo apt-get -y install
+          # Start xvfb daemon
+          export DISPLAY=:99.0
+          /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1400x900x24 -ac +extension GLX +render
+          sleep 5
+          # export python_version
+          PY_VER=${{ matrix.python-version }}
+          echo ::set-output name=python-version::${PY_VER//.}
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          environment-file: continuous_integration/environment.yaml
+          activate-environment: test-environment
+
+      - name: Install unstable dependencies
+        if: matrix.experimental == true
+        run: |
+          python -m pip install \
+          --index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple/ \
+          --trusted-host pypi.anaconda.org \
+          --no-deps --pre --upgrade \
+          matplotlib \
+          numpy \
+          pandas \
+          scipy; \
+          python -m pip install \
+          --no-deps --upgrade \
+          git+https://github.com/dask/dask \
+          git+https://github.com/dask/distributed \
+          git+https://github.com/zarr-developers/zarr \
+          git+https://github.com/Unidata/cftime \
+          git+https://github.com/mapbox/rasterio \
+          git+https://github.com/pydata/bottleneck \
+          git+https://github.com/pydata/xarray;
+
+      - name: Install uwsift
+        run: |
+          pip install --no-deps -e .
+
+      - name: Run unit tests
+        run: |
+          export DISPLAY=:99.0
+          pytest --cov=uwsift uwsift/tests
+
+      - name: Coveralls Parallel
+        uses: AndreMiras/coveralls-python-action@develop
+        with:
+          flag-name: run-${{ matrix.test_number }}
+          parallel: true
+        if: runner.os == 'Linux'
+
+  coveralls:
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: AndreMiras/coveralls-python-action@develop
+        with:
+          parallel-finished: true
+

.github/workflows/deploy.yaml

@@ -0,0 +1,102 @@
+name: Deploy
+# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
+# https://docs.github.com/en/developers/webhooks-and-events/events/github-event-types#pullrequestevent
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.type }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - master
+  release:
+    types:
+      - published
+
+jobs:
+  sdist:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Create sdist
+        shell: bash -l {0}
+        run: python setup.py sdist
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.tar.gz
+
+  bundle:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: "bash -l {0}"
+    continue-on-error: false
+    strategy:
+      fail-fast: true
+      matrix:
+        os: ["windows-latest", "ubuntu-latest", "macos-latest"]
+        python-version: ["3.9"]
+
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      OS: ${{ matrix.os }}
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Install SSH Key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SIFT_SFTP_UPLOAD_KEY }}
+          name: id_rsa_sftp
+          known_hosts: ${{ secrets.SIFT_SFTP_UPLOAD_KNOWN_HOSTS }}
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          environment-file: continuous_integration/environment.yaml
+          activate-environment: test-environment
+
+      - name: Install conda pack
+        run: |
+          conda install conda-pack
+
+      - name: Install uwsift
+        run: |
+          pip install --no-deps .
+
+      - name: Build bundle
+        run: |
+          continuous_integration/build_conda_pack.sh
+
+      # It would be better to only upload if all platforms had succeeded but
+      # that would require uploading the bundles as artifacts which would
+      # quickly hit our temporary storage limits
+      - name: Upload bundle
+        run: |
+          continuous_integration/upload_conda_pack.sh
+
+  upload-sdist:
+    runs-on: ubuntu-latest
+    needs: [sdist, bundle]  # don't deploy unless sdist and bundle building succeeded
+    # publish when a GitHub Release is created
+    if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.UWSIFT_PYPI_TOKEN }}

continuous_integration/build_conda_pack.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -ex
+GIT_TAG="${GITHUB_REF##*/}"
+
+if [[ $GIT_TAG =~ [0-9]+.[0-9]+.[0-9]+ ]]; then
+    # valid tag (use default script options)
+    oflag=""
+else
+    # master branch
+    version=$(python -c "from uwsift import __version__; print(__version__)")
+
+    if [[ "${OS}" == "windows-latest" ]]; then
+        ext="zip"
+        platform="windows"
+    else
+        ext="tar.gz"
+        if [[ "${OS}" == "macos-latest" ]]; then
+            platform="darwin"
+        else
+            platform="linux"
+        fi;
+    fi;
+    oflag="-o SIFT_${version}dev_${platform}_$(date +%Y%m%d_%H%M%S).${ext}"
+fi
+
+python build_conda_pack.py -j -1 $oflag
+ls -l
+
+set +ex

continuous_integration/environment.yaml

@@ -0,0 +1,57 @@
+name: test-environment
+channels:
+  - conda-forge
+dependencies:
+  - appdirs
+  - Cython
+  - imageio
+  - imageio-ffmpeg
+  - matplotlib
+  - numba
+  - numpy
+  - pyproj
+  - pyshp
+  - pyqt
+  - pyqtgraph
+  - satpy
+  - scikit-image
+  - shapely
+  - sqlalchemy
+  - vispy
+  ### Satpy-only Optional Deps
+  - bottleneck
+  - dask
+  - donfig
+  # 2.19.1 seems to cause library linking issues
+  - eccodes>=2.20
+  - fsspec
+  - h5netcdf
+  - h5py
+  - netcdf4
+  - pillow
+  - pooch
+  - pyhdf
+  - pyresample
+  - python-eccodes
+  - python-geotiepoints
+  - pyyaml
+  - rasterio
+  - rioxarray
+  - xarray
+  - zarr
+  ### Development/Test dependencies
+  - coveralls
+  - coverage
+  - codecov
+  - pytest
+  - pytest-cov
+  - pytest-mock
+  - pytest-qt
+  - pip
+  - sphinx
+  ### Pip Dependencies
+  - pip:
+    - trollsift
+    - trollimage
+    - pyspectral
+    - pyorbital

continuous_integration/linux_full_deps_apt.txt

@@ -0,0 +1,7 @@
+libglu1-mesa-dev
+libgl1-mesa-dev
+libxi-dev
+libglfw3-dev
+libgles2-mesa-dev
+libsdl2-2.0-0
+mesa-utils

continuous_integration/upload_conda_pack.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -ex
+
+GIT_TAG="${GITHUB_REF##*/}"
+if [[ $GIT_TAG =~ [0-9]+.[0-9]+.[0-9]+ ]]; then
+    # valid tag
+    odir=""
+else
+    # master branch
+    odir="experimental/"
+fi
+# Upload the new bundle
+curl -k --ftp-create-dirs -T SIFT_*.*.*_*.* --key $HOME/.ssh/id_rsa_sftp sftp://sift@ftp.ssec.wisc.edu/${odir}
+set +e
+# Delete any old
+if [[ $GIT_TAG =~ [0-9]+.[0-9]+.[0-9]+ ]]; then
+    curl -k -l --key $HOME/.ssh/id_rsa_sftp sftp://sift@ftp.ssec.wisc.edu/experimental/ | grep SIFT_*.*.*_*.* | xargs -I{} -- curl -k -v --key $HOME/.ssh/id_rsa_sftp sftp://sift@ftp.ssec.wisc.edu/experimental/ -Q "RM experimental/{}"
+    if [ $? -ne 0 ]; then
+        echo "Failed to delete old experimental SIFT tarballs from FTP server"
+    fi
+fi
+
+set +x