From 3817c32b7b34611492890c843d1d5e9615cefccd Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 23 Apr 2021 13:16:45 -0600 Subject: [PATCH 01/24] initial gh action --- .github/workflows/python-package.yml | 125 +++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..6a4dac1 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,125 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Unittest, flake8, coverage + +# Run action on pull requests +# Run on a published release and push to Pypi +on: + pull_request: + branches: [ main ] + release: + types: [published] + +jobs: + + flake8: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install flake8 + + - name: Lint with flake8 + run: | + flake8 topocalc + + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install coverage coveralls PyYAML + python3 -m pip install -r requirements.txt + + - name: Run coverage + run: | + make coveralls + + + unittest: + needs: [flake8, coverage] + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + - name: Run unittests + run: | + python3 -m unittest -v + + build_dist: + needs: unittest + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + - name: Build dist + run: python setup.py sdist --formats=gztar + + - name: Build wheel + run: python3 setup.py bdist_wheel + + - uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + upload_pypi: + needs: [build_sdist] + runs-on: ubuntu-latest + # upload to PyPI on every tag starting with 'v' + # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') + # alternatively, to publish when a GitHub Release is created, use the following rule: + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} + # To test: repository_url: https://test.pypi.org/legacy/ \ No newline at end of file From e2af84bc01215995adae554cc05443e0624fa9b2 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 23 Apr 2021 13:29:47 -0600 Subject: [PATCH 02/24] on push --- .github/workflows/python-package.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6a4dac1..d9bd7a7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,11 +5,11 @@ name: Unittest, flake8, coverage # Run action on pull requests # Run on a published release and push to Pypi -on: - pull_request: - branches: [ main ] - release: - types: [published] +on: [push] + # pull_request: + # branches: [ main ] + # release: + # types: [published] jobs: From 75b90111913dce1a09ec3a11700976de1abee2e3 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 23 Apr 2021 13:32:16 -0600 Subject: [PATCH 03/24] run some tests --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d9bd7a7..87a539a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -76,7 +76,7 @@ jobs: - name: Run unittests run: | - python3 -m unittest -v + python3 -m unittest -v tests/data/hrrr/test_* build_dist: needs: unittest @@ -106,7 +106,7 @@ jobs: path: dist/*.tar.gz upload_pypi: - needs: [build_sdist] + needs: [build_dist] runs-on: ubuntu-latest # upload to PyPI on every tag starting with 'v' # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') From bbba4bcef50ef4c0ee3da7c9162fb2b4cd554513 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 23 Apr 2021 13:34:21 -0600 Subject: [PATCH 04/24] remove coverage, update flake8 call --- .github/workflows/python-package.yml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 87a539a..412a625 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,31 +30,10 @@ jobs: - name: Lint with flake8 run: | - flake8 topocalc - - coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - python3 -m pip install coverage coveralls PyYAML - python3 -m pip install -r requirements.txt - - - name: Run coverage - run: | - make coveralls - + flake8 weather_forecast_retrieval unittest: - needs: [flake8, coverage] + needs: flake8 strategy: matrix: os: [ubuntu-latest, macos-latest] From 3a87304a7f78e138cc7b384d579d13b3ff6c1ae0 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 23 Apr 2021 13:40:50 -0600 Subject: [PATCH 05/24] setuptools_scm, removed travis, added test requirement extras --- .github/workflows/python-package.yml | 3 +-- .travis.yml | 16 ---------------- requirements.txt | 1 + setup.py | 17 ++++++++--------- weather_forecast_retrieval/__init__.py | 15 --------------- 5 files changed, 10 insertions(+), 42 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 412a625..46660cc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -54,8 +54,7 @@ jobs: python3 -m pip install -r requirements.txt - name: Run unittests - run: | - python3 -m unittest -v tests/data/hrrr/test_* + run: python3 setup.py test -s tests.data.hrrr.test_file_loader build_dist: needs: unittest diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 28cdc14..0000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Config file for automatic testing at travis-ci.org -# This file will be regenerated if you run travis_pypi_setup.py - -language: python -python: - - 3.6 - -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install flake8 - -# command to run tests, e.g. python setup.py test -script: - - make lint - # This won't work because all the dependancies, need the docker image - # - make coverage - diff --git a/requirements.txt b/requirements.txt index 96b997f..4211802 100755 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ cfgrib>=0.9.7.1 beautifulsoup4 pandas numpy +setuptools_scm<4.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 96832c2..a0865da 100644 --- a/setup.py +++ b/setup.py @@ -15,17 +15,8 @@ with open('requirements.txt') as req_file: requirements = req_file.read() -# setup_requirements = [ -# # TODO(scotthavens): put setup requirements (distutils extensions, etc.) here -# ] - -# test_requirements = [ -# # TODO: put package test requirements here -# ] - setup( name='weather_forecast_retrieval', - version='0.6.14', description="Weather forecast retrieval gathers relevant gridded weather " "forecasts to ingest into physically based models for water " "supply forecasts", @@ -54,6 +45,14 @@ 'Programming Language :: Python :: 3.7', ], test_suite='tests', + extras_require={ + 'tests': [ + 'mock', + ], + }, + use_scm_version={ + 'local_scheme': 'node-and-date', + }, scripts=[ 'scripts/run_hrrr_retrieval', 'scripts/run_hrrr_retrieval_dates', diff --git a/weather_forecast_retrieval/__init__.py b/weather_forecast_retrieval/__init__.py index eba7d13..e69de29 100644 --- a/weather_forecast_retrieval/__init__.py +++ b/weather_forecast_retrieval/__init__.py @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Top-level package for Weather Forecast Retrieval.""" - -__author__ = """Scott Havens""" -__email__ = 'scott.havens@ars.usda.gov' -__version__ = '0.6.14' - -# from . import hrrr, hrrr_archive, rap, utils - -from .hrrr import HRRR - -__all__ = [ - HRRR -] From 3feb4f284494b68d8daea17887530f65d422dcee Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 09:25:12 -0600 Subject: [PATCH 06/24] skipping tests --- .github/workflows/python-package.yml | 5 ++++- tests/helpers.py | 4 ++++ tests/test_grib2nc.py | 5 +++++ tests/test_hrrr_preprocessor.py | 14 ++++++++++---- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 46660cc..9d444e2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -54,7 +54,10 @@ jobs: python3 -m pip install -r requirements.txt - name: Run unittests - run: python3 setup.py test -s tests.data.hrrr.test_file_loader + env: + WFR_SKIP_EXTERNAL_REQUEST_TEST: please_skip + WFR_SKIP_ON_GITHUB_ACTIONS: also_skip + run: python3 -m unittest -v build_dist: needs: unittest diff --git a/tests/helpers.py b/tests/helpers.py index 67b1981..8fd3638 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -3,3 +3,7 @@ def skip_external_http_request(): return 'WFR_SKIP_EXTERNAL_REQUEST_TEST' in os.environ + + +def skip_on_github_actions(): + return 'WFR_SKIP_ON_GITHUB_ACTIONS' in os.environ diff --git a/tests/test_grib2nc.py b/tests/test_grib2nc.py index 2547dd2..4b875fc 100644 --- a/tests/test_grib2nc.py +++ b/tests/test_grib2nc.py @@ -1,12 +1,17 @@ import os +import unittest from tests.RME_test_case import RMETestCase +from tests.helpers import skip_on_github_actions from weather_forecast_retrieval.grib2nc import grib2nc class TestGrib2nc(RMETestCase): """Tests for `weather_forecast_retrieval` package.""" + @unittest.skipIf( + skip_on_github_actions(), 'On Github Actions, skipping' + ) def testGrib2nc(self): """ Convert test data to netcdf diff --git a/tests/test_hrrr_preprocessor.py b/tests/test_hrrr_preprocessor.py index 0da7900..10f1589 100644 --- a/tests/test_hrrr_preprocessor.py +++ b/tests/test_hrrr_preprocessor.py @@ -1,10 +1,16 @@ +import unittest + import pandas as pd from tests.RME_test_case import RMETestCase +from tests.helpers import skip_on_github_actions from weather_forecast_retrieval.hrrr import HRRR from weather_forecast_retrieval.hrrr_preprocessor import HRRRPreprocessor +@unittest.skipIf( + skip_on_github_actions(), 'On Github Actions, skipping' +) class TestHRRRPreprocessor(RMETestCase): """ Test cropping HRRR files @@ -16,9 +22,9 @@ class TestHRRRPreprocessor(RMETestCase): start_date = '2018-07-22 01:00' end_date = '2018-07-22 02:00' output_files = [ - 'hrrr.20180722/hrrr.t01z.wrfsfcf01.grib2', - 'hrrr.20180722/hrrr.t02z.wrfsfcf01.grib2', - ] + 'hrrr.20180722/hrrr.t01z.wrfsfcf01.grib2', + 'hrrr.20180722/hrrr.t02z.wrfsfcf01.grib2', + ] def setUp(self): super().setUp() @@ -60,7 +66,7 @@ def test_pre_process(self): output_dir=self.output_path.as_posix(), force_zone_number=self.UTM_ZONE_NUMBER) - self.assertListEqual( + self.assertCountEqual( list(data.keys()), ['air_temp', 'relative_humidity', 'wind_u', 'wind_v', 'precip_int', 'short_wave'] ) From 1aab81001430c776fc611b086466c1d9867e75fc Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 09:27:52 -0600 Subject: [PATCH 07/24] install extras --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9d444e2..e742b60 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -52,6 +52,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt + python3 -m pip install .[tests] - name: Run unittests env: From c6a161333bbf3a42ce2ffc764259c8852f9abb34 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:32:58 -0600 Subject: [PATCH 08/24] fixing merge artifact and more logging for error --- tests/data/hrrr/test_hrrr_gold.py | 2 +- tests/test_hrrr_preprocessor.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/data/hrrr/test_hrrr_gold.py b/tests/data/hrrr/test_hrrr_gold.py index 2457fd9..2fcfdea 100755 --- a/tests/data/hrrr/test_hrrr_gold.py +++ b/tests/data/hrrr/test_hrrr_gold.py @@ -35,7 +35,7 @@ def testHRRRGribLoad(self): """ metadata, data = FileLoader( file_dir=self.hrrr_dir.as_posix(), - config=tests.helpers.LOG_ERROR_CONFIG, + # config=tests.helpers.LOG_ERROR_CONFIG, ).get_saved_data( self.START_DATE, self.END_DATE, diff --git a/tests/test_hrrr_preprocessor.py b/tests/test_hrrr_preprocessor.py index eb81c1c..3089b5a 100644 --- a/tests/test_hrrr_preprocessor.py +++ b/tests/test_hrrr_preprocessor.py @@ -69,10 +69,6 @@ def test_pre_process(self): ) self.assertCountEqual( - << << << < HEAD list(data.keys()), - == == == = - data.keys(), - >>>>>> > main ['air_temp', 'relative_humidity', 'wind_u', 'wind_v', 'precip_int', 'short_wave'] ) From 2c15a23513abfc406a1da425561dde3af45ba63b Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:36:32 -0600 Subject: [PATCH 09/24] print directory --- tests/data/hrrr/test_hrrr_gold.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/hrrr/test_hrrr_gold.py b/tests/data/hrrr/test_hrrr_gold.py index 2fcfdea..8dacc7f 100755 --- a/tests/data/hrrr/test_hrrr_gold.py +++ b/tests/data/hrrr/test_hrrr_gold.py @@ -33,6 +33,7 @@ def testHRRRGribLoad(self): """ Load HRRR data from multiple grib files """ + print(self.hrrr_dir.as_posix()) metadata, data = FileLoader( file_dir=self.hrrr_dir.as_posix(), # config=tests.helpers.LOG_ERROR_CONFIG, From b1424855f6d597bbe5dc8444d7d9ab091791cd8b Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:40:34 -0600 Subject: [PATCH 10/24] looking for files --- .github/workflows/python-package.yml | 8 ++++++++ tests/data/hrrr/test_hrrr_gold.py | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e742b60..cb91d03 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,6 +32,14 @@ jobs: run: | flake8 weather_forecast_retrieval + - name: ls dir + run: | + ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval + ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests + ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME + ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME/gridded + ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME/gridded/hrrr_test + unittest: needs: flake8 strategy: diff --git a/tests/data/hrrr/test_hrrr_gold.py b/tests/data/hrrr/test_hrrr_gold.py index 8dacc7f..2457fd9 100755 --- a/tests/data/hrrr/test_hrrr_gold.py +++ b/tests/data/hrrr/test_hrrr_gold.py @@ -33,10 +33,9 @@ def testHRRRGribLoad(self): """ Load HRRR data from multiple grib files """ - print(self.hrrr_dir.as_posix()) metadata, data = FileLoader( file_dir=self.hrrr_dir.as_posix(), - # config=tests.helpers.LOG_ERROR_CONFIG, + config=tests.helpers.LOG_ERROR_CONFIG, ).get_saved_data( self.START_DATE, self.END_DATE, From d92fb09fd47185d8de0100a14418faf132d8d523 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:41:41 -0600 Subject: [PATCH 11/24] ls hrrr dir --- .github/workflows/python-package.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cb91d03..2ab4e76 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -34,11 +34,7 @@ jobs: - name: ls dir run: | - ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval - ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests - ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME - ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME/gridded - ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME/gridded/hrrr_test + ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME/gridded/hrrr_test/hrrr.20180722 unittest: needs: flake8 From eb997857d02fb57bbd4ab5b657067d124b947d07 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:47:16 -0600 Subject: [PATCH 12/24] logger to debug --- .github/workflows/python-package.yml | 4 ---- tests/helpers.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2ab4e76..e742b60 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,10 +32,6 @@ jobs: run: | flake8 weather_forecast_retrieval - - name: ls dir - run: | - ls /home/runner/work/weather_forecast_retrieval/weather_forecast_retrieval/tests/RME/gridded/hrrr_test/hrrr.20180722 - unittest: needs: flake8 strategy: diff --git a/tests/helpers.py b/tests/helpers.py index 47c0921..fe732b4 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,7 +2,7 @@ LOG_ERROR_CONFIG = { 'logging': { - 'log_level': 'ERROR', + 'log_level': 'DEBUG', } } From 3a972f4e58086ae669eeae503e78c5e68817146a Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:50:59 -0600 Subject: [PATCH 13/24] targeted testing --- .github/workflows/python-package.yml | 42 +++++++++++++++------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e742b60..ec1c7c1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,31 +13,33 @@ on: [push] jobs: - flake8: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' + # flake8: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v2 + + # - name: Set up Python + # uses: actions/setup-python@v2 + # with: + # python-version: '3.7' - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - python3 -m pip install flake8 + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade pip + # python3 -m pip install flake8 - - name: Lint with flake8 - run: | - flake8 weather_forecast_retrieval + # - name: Lint with flake8 + # run: | + # flake8 weather_forecast_retrieval unittest: - needs: flake8 + # needs: flake8 strategy: matrix: - os: [ubuntu-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + # os: [ubuntu-latest, macos-latest] + # python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest] + python-version: [3.9] runs-on: ${{ matrix.os }} steps: @@ -58,7 +60,7 @@ jobs: env: WFR_SKIP_EXTERNAL_REQUEST_TEST: please_skip WFR_SKIP_ON_GITHUB_ACTIONS: also_skip - run: python3 -m unittest -v + run: python3 -m unittest -v tests/data/hrrr/test_hrrr_gold.py build_dist: needs: unittest From 2873a64ad8cc63c714ab07053adbe881576e741f Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 13:59:39 -0600 Subject: [PATCH 14/24] install eccodes --- .github/workflows/python-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ec1c7c1..bd593dd 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -50,6 +50,10 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Ubuntu install eccodes + if: ${{ matrix.os == ubuntu-latest }} + run: apt install libeccodes-tools + - name: Install dependencies run: | python3 -m pip install --upgrade pip From 9c4dd20fa5ac374ec736234d47ad521f4c876bf4 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:02:55 -0600 Subject: [PATCH 15/24] install eccodes --- .github/workflows/python-package.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index bd593dd..f8deb36 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -50,9 +50,17 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Ubuntu install eccodes - if: ${{ matrix.os == ubuntu-latest }} - run: apt install libeccodes-tools + - name: Install eccodes + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + apt install libeccodes-tools + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install eccodes + else + echo "$RUNNER_OS not supported" + exit 1 + fi + shell: bash - name: Install dependencies run: | From 46da7dff713f507cb2f1601b3e0cdd3caf199465 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:09:44 -0600 Subject: [PATCH 16/24] fix syntax --- .github/workflows/python-package.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f8deb36..797d35f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -52,15 +52,15 @@ jobs: - name: Install eccodes run: | - if [ "$RUNNER_OS" == "Linux" ]; then + if [ "$RUNNER_OS" == "Linux" ]; then apt install libeccodes-tools - elif [ "$RUNNER_OS" == "macOS" ]; then + elif [ "$RUNNER_OS" == "macOS" ]; then brew install eccodes - else + else echo "$RUNNER_OS not supported" exit 1 - fi - shell: bash + fi + shell: bash - name: Install dependencies run: | From 910c21208db0803162e24079cf4f52d8c15a16a7 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:11:16 -0600 Subject: [PATCH 17/24] fix ubuntu install --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 797d35f..87d5d2f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -53,7 +53,7 @@ jobs: - name: Install eccodes run: | if [ "$RUNNER_OS" == "Linux" ]; then - apt install libeccodes-tools + sudo apt-get install -y libeccodes-tools elif [ "$RUNNER_OS" == "macOS" ]; then brew install eccodes else From 2710665eb455aa520d18f50fc5556dad9271f333 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:15:29 -0600 Subject: [PATCH 18/24] add osx, move release to a new workflow --- .github/workflows/new_pypi_release.yml | 57 ++++++++++++++++++++++++++ .github/workflows/python-package.yml | 25 +---------- 2 files changed, 58 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/new_pypi_release.yml diff --git a/.github/workflows/new_pypi_release.yml b/.github/workflows/new_pypi_release.yml new file mode 100644 index 0000000..e2ecb47 --- /dev/null +++ b/.github/workflows/new_pypi_release.yml @@ -0,0 +1,57 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Release to Pypi + +# Run on a published release and push to Pypi +on: + release: + types: [published] + +jobs: + + build_dist: + needs: unittest + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + - name: Build dist + run: python setup.py sdist --formats=gztar + + - name: Build wheel + run: python3 setup.py bdist_wheel + + - uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + upload_pypi: + needs: [build_dist] + runs-on: ubuntu-latest + # upload to PyPI on every tag starting with 'v' + # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') + # alternatively, to publish when a GitHub Release is created, use the following rule: + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} + # To test: repository_url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 87d5d2f..608403c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,12 +4,9 @@ name: Unittest, flake8, coverage # Run action on pull requests -# Run on a published release and push to Pypi on: [push] # pull_request: # branches: [ main ] - # release: - # types: [published] jobs: @@ -36,9 +33,8 @@ jobs: # needs: flake8 strategy: matrix: - # os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest] # python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest] python-version: [3.9] runs-on: ${{ matrix.os }} @@ -100,22 +96,3 @@ jobs: - uses: actions/upload-artifact@v2 with: path: dist/*.tar.gz - - upload_pypi: - needs: [build_dist] - runs-on: ubuntu-latest - # upload to PyPI on every tag starting with 'v' - # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') - # alternatively, to publish when a GitHub Release is created, use the following rule: - if: github.event_name == 'release' && github.event.action == 'published' - steps: - - uses: actions/download-artifact@v2 - with: - name: artifact - path: dist - - - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.pypi_password }} - # To test: repository_url: https://test.pypi.org/legacy/ \ No newline at end of file From 6a9d73f915973155e7bdbcacf9c46c79177edbea Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:43:38 -0600 Subject: [PATCH 19/24] full pipeline --- .github/workflows/python-package.yml | 39 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 608403c..c816a45 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,32 +10,31 @@ on: [push] jobs: - # flake8: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v2 - - # - name: Set up Python - # uses: actions/setup-python@v2 - # with: - # python-version: '3.7' + flake8: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' - # - name: Install dependencies - # run: | - # python3 -m pip install --upgrade pip - # python3 -m pip install flake8 + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install flake8 - # - name: Lint with flake8 - # run: | - # flake8 weather_forecast_retrieval + - name: Lint with flake8 + run: | + flake8 weather_forecast_retrieval unittest: - # needs: flake8 + needs: flake8 strategy: matrix: os: [ubuntu-latest, macos-latest] - # python-version: [3.6, 3.7, 3.8, 3.9] - python-version: [3.9] + python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} steps: @@ -84,7 +83,7 @@ jobs: - name: Install dependencies run: | - python3 -m pip install --upgrade pip + python3 -m pip install --upgrade pip wheel python3 -m pip install -r requirements.txt - name: Build dist From dc8025bb5e69c7ee1f5060caede5c12eddcb86a5 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:51:17 -0600 Subject: [PATCH 20/24] clean up workflows --- .github/workflows/new_pypi_release.yml | 3 +-- .github/workflows/python-package.yml | 37 ++++---------------------- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/.github/workflows/new_pypi_release.yml b/.github/workflows/new_pypi_release.yml index e2ecb47..bfb477a 100644 --- a/.github/workflows/new_pypi_release.yml +++ b/.github/workflows/new_pypi_release.yml @@ -11,7 +11,6 @@ on: jobs: build_dist: - needs: unittest name: Build source distribution runs-on: ubuntu-latest steps: @@ -24,7 +23,7 @@ jobs: - name: Install dependencies run: | - python3 -m pip install --upgrade pip + python3 -m pip install --upgrade pip wheel python3 -m pip install -r requirements.txt - name: Build dist diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c816a45..2612bc9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,12 +1,12 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Unittest, flake8, coverage +name: Unittest, flake8 # Run action on pull requests -on: [push] - # pull_request: - # branches: [ main ] +on: + pull_request: + branches: [ main ] jobs: @@ -67,31 +67,4 @@ jobs: env: WFR_SKIP_EXTERNAL_REQUEST_TEST: please_skip WFR_SKIP_ON_GITHUB_ACTIONS: also_skip - run: python3 -m unittest -v tests/data/hrrr/test_hrrr_gold.py - - build_dist: - needs: unittest - name: Build source distribution - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.8' - - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip wheel - python3 -m pip install -r requirements.txt - - - name: Build dist - run: python setup.py sdist --formats=gztar - - - name: Build wheel - run: python3 setup.py bdist_wheel - - - uses: actions/upload-artifact@v2 - with: - path: dist/*.tar.gz + run: python3 -m unittest -v tests/data/hrrr/test_hrrr_gold.py \ No newline at end of file From aea3c1c8a3fdf29bedd916a3ffa63ad046749a88 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 14:54:53 -0600 Subject: [PATCH 21/24] isort, rename workflow --- .github/workflows/{python-package.yml => unittest.yml} | 2 +- tests/data/hrrr/test_config_file.py | 1 + tests/data/hrrr/test_file_loader.py | 3 ++- tests/data/hrrr/test_http_retrieval.py | 2 +- tests/helpers.py | 2 +- tests/test_grib2nc.py | 2 +- tests/test_hrrr_archive.py | 2 +- tests/test_hrrr_preprocessor.py | 3 +-- weather_forecast_retrieval/hrrr_archive.py | 1 + 9 files changed, 10 insertions(+), 8 deletions(-) rename .github/workflows/{python-package.yml => unittest.yml} (96%) diff --git a/.github/workflows/python-package.yml b/.github/workflows/unittest.yml similarity index 96% rename from .github/workflows/python-package.yml rename to .github/workflows/unittest.yml index 2612bc9..970c3db 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/unittest.yml @@ -67,4 +67,4 @@ jobs: env: WFR_SKIP_EXTERNAL_REQUEST_TEST: please_skip WFR_SKIP_ON_GITHUB_ACTIONS: also_skip - run: python3 -m unittest -v tests/data/hrrr/test_hrrr_gold.py \ No newline at end of file + run: python3 -m unittest -v \ No newline at end of file diff --git a/tests/data/hrrr/test_config_file.py b/tests/data/hrrr/test_config_file.py index b135b8b..d0dc206 100644 --- a/tests/data/hrrr/test_config_file.py +++ b/tests/data/hrrr/test_config_file.py @@ -2,6 +2,7 @@ import unittest import pandas as pd + import tests.helpers from weather_forecast_retrieval.data.hrrr.config_file import ConfigFile diff --git a/tests/data/hrrr/test_file_loader.py b/tests/data/hrrr/test_file_loader.py index 4f6cff4..c0453ab 100755 --- a/tests/data/hrrr/test_file_loader.py +++ b/tests/data/hrrr/test_file_loader.py @@ -2,8 +2,9 @@ import unittest import mock -import tests.helpers import xarray + +import tests.helpers from tests.RME import RMETestCase from weather_forecast_retrieval.data.hrrr.file_loader import FileLoader from weather_forecast_retrieval.data.hrrr.grib_file import GribFile diff --git a/tests/data/hrrr/test_http_retrieval.py b/tests/data/hrrr/test_http_retrieval.py index 33d0a40..2c9da9f 100644 --- a/tests/data/hrrr/test_http_retrieval.py +++ b/tests/data/hrrr/test_http_retrieval.py @@ -3,8 +3,8 @@ import pandas as pd -from tests.RME import RMETestCase from tests.helpers import skip_external_http_request +from tests.RME import RMETestCase from weather_forecast_retrieval.data.hrrr import HttpRetrieval diff --git a/tests/helpers.py b/tests/helpers.py index fe732b4..47c0921 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,7 +2,7 @@ LOG_ERROR_CONFIG = { 'logging': { - 'log_level': 'DEBUG', + 'log_level': 'ERROR', } } diff --git a/tests/test_grib2nc.py b/tests/test_grib2nc.py index f0c2fe6..6084fda 100644 --- a/tests/test_grib2nc.py +++ b/tests/test_grib2nc.py @@ -1,8 +1,8 @@ import os import unittest -from tests.RME import RMETestCase from tests.helpers import skip_on_github_actions +from tests.RME import RMETestCase from weather_forecast_retrieval.grib2nc import grib2nc diff --git a/tests/test_hrrr_archive.py b/tests/test_hrrr_archive.py index f03ae66..4e9dc50 100644 --- a/tests/test_hrrr_archive.py +++ b/tests/test_hrrr_archive.py @@ -7,8 +7,8 @@ import numpy as np import pandas as pd -from tests.RME import RMETestCase from tests.helpers import skip_external_http_request +from tests.RME import RMETestCase from weather_forecast_retrieval import hrrr_archive diff --git a/tests/test_hrrr_preprocessor.py b/tests/test_hrrr_preprocessor.py index 3089b5a..a3014ee 100644 --- a/tests/test_hrrr_preprocessor.py +++ b/tests/test_hrrr_preprocessor.py @@ -2,9 +2,8 @@ import pandas as pd -from tests.RME import RMETestCase from tests.helpers import skip_on_github_actions - +from tests.RME import RMETestCase from weather_forecast_retrieval.data.hrrr import FileLoader from weather_forecast_retrieval.hrrr_preprocessor import HRRRPreprocessor diff --git a/weather_forecast_retrieval/hrrr_archive.py b/weather_forecast_retrieval/hrrr_archive.py index c50999b..2ccd75c 100755 --- a/weather_forecast_retrieval/hrrr_archive.py +++ b/weather_forecast_retrieval/hrrr_archive.py @@ -92,6 +92,7 @@ def download_url(fname, OUTDIR, logger, file_day, model='hrrr', field='sfc'): check_this = requests.head(URL) file_size = int(check_this.headers['content-length']) + success = False try: if file_size > 10000: logger.info("Downloading: {}".format(URL)) From c8bcd1e941ac6df90d0a98f2c89ba220f0353ab9 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 15:11:47 -0600 Subject: [PATCH 22/24] updating setup.py --- setup.py | 12 +++--------- weather_forecast_retrieval/__init__.py | 6 ++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index a0865da..264bf5c 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,8 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""The setup script.""" - -# # from setuptools import setup, find_packages from setuptools import setup with open('README.md') as readme_file: readme = readme_file.read() -# with open('HISTORY.rst') as history_file: -# history = history_file.read() - with open('requirements.txt') as req_file: requirements = req_file.read() @@ -53,6 +44,9 @@ use_scm_version={ 'local_scheme': 'node-and-date', }, + setup_requires=[ + 'setuptools_scm' + ], scripts=[ 'scripts/run_hrrr_retrieval', 'scripts/run_hrrr_retrieval_dates', diff --git a/weather_forecast_retrieval/__init__.py b/weather_forecast_retrieval/__init__.py index e69de29..e890949 100644 --- a/weather_forecast_retrieval/__init__.py +++ b/weather_forecast_retrieval/__init__.py @@ -0,0 +1,6 @@ +from pkg_resources import DistributionNotFound, get_distribution + +try: + __version__ = get_distribution(__name__).version +except DistributionNotFound: + __version__ = 'unknown' From 89f3de7572c0bebdd43b094352d9edbc53fec0ef Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 26 Apr 2021 15:12:41 -0600 Subject: [PATCH 23/24] another update --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 264bf5c..c3f49c5 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,11 @@ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'Natural Language :: English', + 'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], test_suite='tests', extras_require={ From 5c7a0b73704ce614f853cc9239922817b144d55e Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Tue, 27 Apr 2021 09:36:30 -0600 Subject: [PATCH 24/24] updated list assert --- tests/test_hrrr_preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_hrrr_preprocessor.py b/tests/test_hrrr_preprocessor.py index a3014ee..666087f 100644 --- a/tests/test_hrrr_preprocessor.py +++ b/tests/test_hrrr_preprocessor.py @@ -68,6 +68,6 @@ def test_pre_process(self): ) self.assertCountEqual( - list(data.keys()), + data.keys(), ['air_temp', 'relative_humidity', 'wind_u', 'wind_v', 'precip_int', 'short_wave'] )