Develop one steps (#236)

* Feature/one step save baseline (#193)

This adds several features to the one-step pipeline

- big zarr. Everything is stored as one zarr file
- saves physics outputs
- some refactoring of the job submission.

Sample output: https://gist.github.com/nbren12/84536018dafef01ba5eac0354869fb67

* save lat/lon grid variables from sfc_dt_atmos (#204)

* save lat/lon grid variables from sfc_dt_atmos

* Feature/use onestep zarr train data (#207)

Use the big zarr from the one step workflow as input to the create training data pipeline

* One-step sfc variables time alignment (#214)

This makes the diagnostics variables appended to the big zarr have the appropriate step and forecast_time dimensions, just as the variables extracted by the wrapper do.

* One step zarr fill values (#223)

This accomplishes two things: 1) preventing true model 0s from being cast to NaNs in the one-step big zarr output, and 2) initializing big zarr arrays with NaNs via full so that if they are not filled in due to a failed timestep or other reason, it is more apparent than using empty which produces arbitrary output.

* adjustments to be able to run workflows in dev branch (#218)

Remove references to hard coded dims and data variables or imports from vcm.cubedsphere.constants, replace with arguments.
Can provide coords and dims as args for mappable var

* One steps start index (#231)

Allows for starting the one-step jobs at the specified index in the timestep list to allow for testing/avoiding spinup timesteps

* Dev fix/integration tests (#234)

* change order of required args so output is last

* fix arg for onestep input to be dir containing big zarr

* update end to end integration test ymls

* prognostic run adjustments

* Improved fv3 logging (#225)

This PR introduces several improvements to the logging capability of our prognostic run image

- include upstream changes to disable output capturing in `fv3config.fv3run`
- Add `capture_fv3gfs_func` function. When called this capture the raw fv3gfs outputs and re-emit it as DEBUG level logging statements that can more easily be filtered.
- Refactor `runtime` to `external/runtime/runtime`. This was easy since it did not depend on any other module in fv3net. (except implicitly the code in `fv3net.regression` which is imported when loading the sklearn model with pickle).
- updates fv3config to master

* manually merge in the refactor from master while keeping new names from develop (#237)

* lint

* remove logging from testing

* Dev fix/arg order (#238)

* update history

* fix positional args

* fix function args

* update history

* linting

Co-authored-by: Anna Kwa <annak@vulcan.com>
Co-authored-by: brianhenn <brianhenn@gmail.com>

.dockerignore

@@ -32,4 +32,5 @@ tox.ini
 vcm-ml-data
 dataflow/*/env
 external/vcm/venv
-Dockerfile
+Dockerfile
+outdir/

.environment-scripts/install_local_packages.sh

@@ -4,8 +4,17 @@ CONDA_ENV=$1
 
 source activate $CONDA_ENV
 
-local_packages_to_install=(. external/fv3config external/vcm external/vcm/external/mappm)
+local_packages_to_install=(. external/fv3config external/vcm external/vcm/external/mappm )
 for package  in ${local_packages_to_install[@]}
 do
   pip install --no-deps -e $package
-done
+done
+
+poetry_packages=( external/runtime )
+for package in ${poetry_packages[@]}
+do
+  (
+    cd $package
+    conda develop .
+  )
+done

HISTORY.rst

@@ -17,6 +17,10 @@ latest
 * Add `nudge_to_obs` module to `kube_jobs`, which helps with the configuration of FV3GFS model runs that are nudged towards GFS analysis
 * Add public function: vcm.convert_timestamps
 * Add pipeline to load C384 restart data into a zarr
+* One step run workflow outputs a single zarr as output (instead of individual directories for each timestep), downstream workflows are adjusted to use this data format
+* Train data pipeline and offline diagnostics workflow read in variable names information from yaml provided to python
+* Force load data in diagnostics workflow before compute and plot
+* Improved logging when running FV3 model
 
 
 0.1.1 (2020-03-25)

Makefile

@@ -3,8 +3,7 @@
 #################################################################################
 # GLOBALS                                                                       #
 #################################################################################
-
-VERSION ?= v0.1.1
+VERSION ?= $(shell git rev-parse HEAD)
 ENVIRONMENT_SCRIPTS = .environment-scripts
 PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 BUCKET = [OPTIONAL] your-bucket-for-syncing-data (do not include 's3://')
@@ -63,6 +62,9 @@ run_integration_tests:
 	./tests/end_to_end_integration/.test_run_scripts/prepare_integration_test_configs.sh $(VERSION)
 	./tests/end_to_end_integration/.test_run_scripts/run_integration_with_wait.sh
 
+test:
+	pytest external/* tests
+
 ## Make Dataset
 .PHONY: data update_submodules create_environment overwrite_baseline_images
 data:

docker/prognostic_run/Dockerfile

@@ -1,8 +1,7 @@
 FROM us.gcr.io/vcm-ml/fv3gfs-python:v0.3.1
 
-
 COPY docker/prognostic_run/requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
+RUN pip3 install wheel && pip3 install -r /tmp/requirements.txt
 
 # cache external package installation
 COPY external/fv3config /fv3net/external/fv3config
@@ -11,3 +10,4 @@ RUN pip3 install -e /fv3net/external/vcm -e /fv3net/external/fv3config
 
 COPY . /fv3net
 RUN pip3 install --no-deps -e /fv3net
+ENV PYTHONPATH=/fv3net/external/runtime:${PYTHONPATH}

docker/prognostic_run/requirements.txt

@@ -1,7 +1,102 @@
+alabaster==0.7.12
+appdirs==1.4.3
+asciitree==0.3.3
+atomicwrites==1.1.5
+attrs==18.2.0
+Babel==2.8.0
+backoff==1.10.0
+cachetools==4.0.0
+certifi==2019.11.28
+cftime==1.1.0
+chardet==3.0.4
+click==7.1.1
+cycler==0.10.0
+Cython==0.29.10
+dask==2.10.1
+decorator==4.4.2
+docrep==0.2.7
+docutils==0.16
+entrypoints==0.3
+f90nml==1.1.2
+fasteners==0.15
+fsspec==0.6.0
+future==0.18.2
+gcsfs==0.4.0
+gitdb==4.0.2
+GitPython==3.1.0
+google-api-core==1.16.0
+google-auth==1.11.2
+google-auth-oauthlib==0.4.1
+google-cloud-core==1.3.0
+google-cloud-storage==1.23.0
+google-resumable-media==0.5.0
+googleapis-common-protos==1.51.0
+h5netcdf==0.8.0
+h5py==2.10.0
+idna==2.8
+imageio==2.8.0
+imagesize==1.2.0
+importlib-metadata==0.19
+intake==0.5.5
+ipython-genutils==0.2.0
+Jinja2==2.10.3
+joblib==0.14.1
+kiwisolver==1.2.0
+kubernetes==10.0.1
+llvmlite==0.31.0
+MarkupSafe==1.1.1
+matplotlib==3.2.1
+MetPy==0.12.0
+monotonic==1.5
+more-itertools==4.2.0
+mpi4py==3.0.2
+msgpack==1.0.0
+netCDF4==1.4.2
+networkx==2.4
+numba==0.48.0
+numcodecs==0.6.4
+numpy==1.16.2
+oauthlib==3.1.0
+packaging==20.1
+pandas==1.0.1
+Pillow==7.1.1
+Pint==0.11
+pluggy==0.12.0
+pooch==1.0.0
+protobuf==3.11.3
+py==1.8.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+Pygments==2.5.2
+pyparsing==2.4.6
+pytest==5.2.2
+python-dateutil==2.8.1
+pytz==2019.3
+PyWavelets==1.1.1
+PyYAML==5.1.2
+requests==2.22.0
+requests-oauthlib==1.3.0
+rsa==4.0
+scikit-image==0.16.2
 scikit-learn==0.22.1
-dask
-joblib
-zarr
-scikit-image
-google-cloud-logging
-backoff
+scipy==1.4.1
+six==1.12.0
+smmap==3.0.1
+snowballstemmer==2.0.0
+Sphinx==2.2.0
+sphinx-rtd-theme==0.4.3
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+toolz==0.10.0
+traitlets==4.3.3
+urllib3==1.25.8
+wcwidth==0.1.8
+websocket-client==0.57.0
+xarray==0.13.0
+xgcm==0.3.0
+zarr==2.3.2
+zipp==0.5.2

environment.yml

@@ -47,7 +47,9 @@ dependencies:
   - bokeh=1.4
   - backoff
   - pip:
+    - poetry
     - gsutil
     - nc-time-axis>=1.2.0
+    - gitpython
     - bump2version>=0.5.11
     - yq