From 89705f14f4b77c380fffb2729e94d77f47544234 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Tue, 4 Jan 2022 13:09:03 -0500 Subject: [PATCH 01/11] First pass at distributed nightly builds --- .github/workflows/conda.yml | 67 +++++++++++++++++++++ continuous_integration/recipe/meta.yaml | 78 +++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 .github/workflows/conda.yml create mode 100644 continuous_integration/recipe/meta.yaml diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 00000000000..fffd532135c --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,67 @@ +name: Conda build +on: + push: + branches: + - main + pull_request: + +# When this workflow is queued, automatically cancel any previous running +# or pending jobs from the same branch +concurrency: + group: conda-${{ github.head_ref }} + cancel-in-progress: true + +# Required shell entrypoint to have properly activated conda environments +defaults: + run: + shell: bash -l {0} + +jobs: + conda: + name: Build (and upload) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + use-mamba: true + python-version: 3.8 + - name: Install dependencies + run: | + mamba install boa conda-verify + + which python + pip list + mamba list + - name: Build conda package + run: | + # suffix for nightly package versions + export VERSION_SUFFIX=a`date +%y%m%d` + + conda mambabuild continuous_integration/recipe \ + --no-anaconda-upload \ + --output-folder build + - name: Upload conda package + if: | + github.event_name == 'push' + && github.ref == 'refs/heads/main' + && github.repository == 'dask/distributed' + env: + ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }} + run: | + # convert package to other architectures + cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \ + -p osx-arm64 \ + -p linux-64 \ + -p linux-ppc64le \ + -p linux-aarch64 \ + -p win-64 + + # install anaconda for upload + mamba install anaconda-client + + anaconda upload --label dev **.tar.bz2 diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml new file mode 100644 index 00000000000..98445785c9e --- /dev/null +++ b/continuous_integration/recipe/meta.yaml @@ -0,0 +1,78 @@ +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set py_version=environ.get('CONDA_PY', 36) %} + + +package: + name: distributed + version: {{ version }} + +source: + git_url: ../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script: {{ PYTHON }} -m pip install . -vv + entry_points: + - dask-scheduler = distributed.cli.dask_scheduler:go + - dask-ssh = distributed.cli.dask_ssh:go + - dask-worker = distributed.cli.dask_worker:go + +requirements: + build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + host: + - python + - pip + + run: + - python + - click >=6.6 + - cloudpickle >=1.5.0 + - cytoolz >=0.8.2 + - dask-core {{ version }} + - jinja2 + - msgpack-python >=0.6.0 + - psutil >=5.0 + - pyyaml + - sortedcontainers !=2.0.0,!=2.0.1 + - tblib >=1.6.0 + - toolz >=0.8.2 + - tornado >=5 # [py<38] + - tornado >=6.0.3 # [py>=38] + - zict >=0.1.3 + - setuptools + + run_constrained: + - openssl !=1.1.1e + +test: + imports: + - distributed + - distributed.cli + - distributed.comm + - distributed.deploy + - distributed.diagnostics + - distributed.protocol + commands: + - pip check + - dask-scheduler --help + - dask-ssh --help + - dask-worker --help + requires: + - pip + +about: + home: https://distributed.dask.org + summary: Distributed scheduler for Dask + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE.txt + description: | + Distributed is a lightweight library for distributed computing in Python. + It extends both the concurrent.futures and dask APIs to moderate sized + clusters. + doc_url: https://distributed.dask.org + doc_source_url: https://github.com/dask/distributed/blob/master/docs/source/index.rst + dev_url: https://github.com/dask/distributed From f3d86a0d44e92605e2fb6c953f19a3b3c45e6e70 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 10 Jan 2022 11:25:45 -0500 Subject: [PATCH 02/11] Pin dask-core dep to latest nightly build --- .github/workflows/conda.yml | 4 ++++ continuous_integration/recipe/meta.yaml | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index fffd532135c..b5934efec5a 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -42,7 +42,11 @@ jobs: # suffix for nightly package versions export VERSION_SUFFIX=a`date +%y%m%d` + # latest dask-core nightly build + export DASK_BUILD=`conda search --override-channels -c dask/label/dev dask-core | tail -n 1 | awk '{print $3}'` + conda mambabuild continuous_integration/recipe \ + --channel dask/label/dev \ --no-anaconda-upload \ --output-folder build - name: Upload conda package diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml index 98445785c9e..03a45f45910 100644 --- a/continuous_integration/recipe/meta.yaml +++ b/continuous_integration/recipe/meta.yaml @@ -1,4 +1,5 @@ {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set dask_build = environ.get('DASK_BUILD', '') %} {% set py_version=environ.get('CONDA_PY', 36) %} @@ -31,7 +32,7 @@ requirements: - click >=6.6 - cloudpickle >=1.5.0 - cytoolz >=0.8.2 - - dask-core {{ version }} + - dask-core *={{ dask_build }} - jinja2 - msgpack-python >=0.6.0 - psutil >=5.0 From 1ade91a46216957468d4d696e25ad10bb5dc1d58 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 10 Jan 2022 11:46:50 -0500 Subject: [PATCH 03/11] Add dask metapackage recipe --- .github/workflows/conda.yml | 15 +++-- continuous_integration/recipes/dask/meta.yaml | 55 +++++++++++++++++++ .../{recipe => recipes/distributed}/meta.yaml | 2 +- 3 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 continuous_integration/recipes/dask/meta.yaml rename continuous_integration/{recipe => recipes/distributed}/meta.yaml (99%) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index b5934efec5a..7c65730ada3 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -37,15 +37,22 @@ jobs: which python pip list mamba list - - name: Build conda package + - name: Build conda packages run: | - # suffix for nightly package versions + # suffix for pre-release package versions export VERSION_SUFFIX=a`date +%y%m%d` - # latest dask-core nightly build + # latest dask-core pre-release build export DASK_BUILD=`conda search --override-channels -c dask/label/dev dask-core | tail -n 1 | awk '{print $3}'` - conda mambabuild continuous_integration/recipe \ + # distributed pre-release build + conda mambabuild continuous_integration/recipes/distributed \ + --channel dask/label/dev \ + --no-anaconda-upload \ + --output-folder build + + # dask pre-release build + conda mambabuild continuous_integration/recipes/dask \ --channel dask/label/dev \ --no-anaconda-upload \ --output-folder build diff --git a/continuous_integration/recipes/dask/meta.yaml b/continuous_integration/recipes/dask/meta.yaml new file mode 100644 index 00000000000..0f7e62984ff --- /dev/null +++ b/continuous_integration/recipes/dask/meta.yaml @@ -0,0 +1,55 @@ +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set dask_build = environ.get('DASK_BUILD', '') %} +{% set py_version=environ.get('CONDA_PY', 36) %} + + +package: + name: dask + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + noarch: python + +requirements: + host: + - python >=3.7 + run: + - python >=3.7 + - dask-core *={{ dask_build }} + - distributed {{ version }} + - cytoolz >=0.8.2 + - numpy >=1.18 + - pandas >=1.0 + - bokeh >=2.1.1 + - jinja2 + + run_constrained: + - openssl !=1.1.1e + +test: + imports: + - dask + - dask.array + - dask.bag + - dask.bytes + - dask.dataframe + - dask.dataframe.tseries + - dask.delayed + - dask.diagnostics + - dask.distributed + - distributed + +about: + home: https://dask.org/ + summary: Parallel PyData with Task Scheduling + license: BSD-3-Clause + license_file: LICENSE.txt + description: | + Dask is a flexible parallel computing library for analytics. + doc_url: https://dask.org/ + dev_url: https://github.com/dask/dask diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipes/distributed/meta.yaml similarity index 99% rename from continuous_integration/recipe/meta.yaml rename to continuous_integration/recipes/distributed/meta.yaml index 03a45f45910..02e526c6e99 100644 --- a/continuous_integration/recipe/meta.yaml +++ b/continuous_integration/recipes/distributed/meta.yaml @@ -8,7 +8,7 @@ package: version: {{ version }} source: - git_url: ../.. + git_url: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} From b479c7f7467d78d9e2a4dc4e3ce1777dee863c31 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:05:10 -0500 Subject: [PATCH 04/11] Try to pin distributed dep by build --- continuous_integration/recipes/dask/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/recipes/dask/meta.yaml b/continuous_integration/recipes/dask/meta.yaml index 0f7e62984ff..d8292765367 100644 --- a/continuous_integration/recipes/dask/meta.yaml +++ b/continuous_integration/recipes/dask/meta.yaml @@ -21,7 +21,7 @@ requirements: run: - python >=3.7 - dask-core *={{ dask_build }} - - distributed {{ version }} + - distributed {{ version }}=py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - cytoolz >=0.8.2 - numpy >=1.18 - pandas >=1.0 From 55f91094aca9b97fb0133a36ac136c2708c85185 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:15:26 -0500 Subject: [PATCH 05/11] Pin dask dep by version and build --- .github/workflows/conda.yml | 1 + continuous_integration/recipes/dask/meta.yaml | 3 ++- continuous_integration/recipes/distributed/meta.yaml | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 7c65730ada3..2b71445902b 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -43,6 +43,7 @@ jobs: export VERSION_SUFFIX=a`date +%y%m%d` # latest dask-core pre-release build + export DASK_VERSION=`conda search --override-channels -c dask/label/dev dask-core | tail -n 1 | awk '{print $2}'` export DASK_BUILD=`conda search --override-channels -c dask/label/dev dask-core | tail -n 1 | awk '{print $3}'` # distributed pre-release build diff --git a/continuous_integration/recipes/dask/meta.yaml b/continuous_integration/recipes/dask/meta.yaml index d8292765367..10d81f5bbd8 100644 --- a/continuous_integration/recipes/dask/meta.yaml +++ b/continuous_integration/recipes/dask/meta.yaml @@ -1,4 +1,5 @@ {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %} {% set dask_build = environ.get('DASK_BUILD', '') %} {% set py_version=environ.get('CONDA_PY', 36) %} @@ -20,7 +21,7 @@ requirements: - python >=3.7 run: - python >=3.7 - - dask-core *={{ dask_build }} + - dask-core {{ dask_version }}={{ dask_build }} - distributed {{ version }}=py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - cytoolz >=0.8.2 - numpy >=1.18 diff --git a/continuous_integration/recipes/distributed/meta.yaml b/continuous_integration/recipes/distributed/meta.yaml index 02e526c6e99..8ec04bc553c 100644 --- a/continuous_integration/recipes/distributed/meta.yaml +++ b/continuous_integration/recipes/distributed/meta.yaml @@ -1,4 +1,5 @@ {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %} {% set dask_build = environ.get('DASK_BUILD', '') %} {% set py_version=environ.get('CONDA_PY', 36) %} @@ -32,7 +33,7 @@ requirements: - click >=6.6 - cloudpickle >=1.5.0 - cytoolz >=0.8.2 - - dask-core *={{ dask_build }} + - dask-core {{ dask_version }}={{ dask_build }} - jinja2 - msgpack-python >=0.6.0 - psutil >=5.0 From 3abe94e42fadade3764f16fa955762d52745ff27 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:26:41 -0500 Subject: [PATCH 06/11] Minor changes to anaconda upload step --- .github/workflows/conda.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 2b71445902b..11ba8d9e091 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -57,7 +57,7 @@ jobs: --channel dask/label/dev \ --no-anaconda-upload \ --output-folder build - - name: Upload conda package + - name: Upload conda packages if: | github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -65,10 +65,9 @@ jobs: env: ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }} run: | - # convert package to other architectures + # convert distributed to other architectures cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \ -p osx-arm64 \ - -p linux-64 \ -p linux-ppc64le \ -p linux-aarch64 \ -p win-64 From 9dafda2543a9c16cbc312b98bb4953eee6475398 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Tue, 11 Jan 2022 15:14:20 -0500 Subject: [PATCH 07/11] Update doc source URL Co-authored-by: jakirkham --- continuous_integration/recipes/distributed/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/recipes/distributed/meta.yaml b/continuous_integration/recipes/distributed/meta.yaml index 8ec04bc553c..b86e5f1a85f 100644 --- a/continuous_integration/recipes/distributed/meta.yaml +++ b/continuous_integration/recipes/distributed/meta.yaml @@ -76,5 +76,5 @@ about: It extends both the concurrent.futures and dask APIs to moderate sized clusters. doc_url: https://distributed.dask.org - doc_source_url: https://github.com/dask/distributed/blob/master/docs/source/index.rst + doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst dev_url: https://github.com/dask/distributed From 0ab4191f188bfb45bcac8f9f85d057727bd117d4 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 12 Jan 2022 10:23:12 -0500 Subject: [PATCH 08/11] Only run conda search command once --- .github/workflows/conda.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 11ba8d9e091..d4f04ff6a6b 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -42,9 +42,12 @@ jobs: # suffix for pre-release package versions export VERSION_SUFFIX=a`date +%y%m%d` - # latest dask-core pre-release build - export DASK_VERSION=`conda search --override-channels -c dask/label/dev dask-core | tail -n 1 | awk '{print $2}'` - export DASK_BUILD=`conda search --override-channels -c dask/label/dev dask-core | tail -n 1 | awk '{print $3}'` + # conda search for the latest dask-core pre-release + arr=($(conda search --override-channels -c dask/label/dev dask-core | tail -n 1)) + + # extract dask-core pre-release version / build + export DASK_VERSION=${arr[1]} + export DASK_BUILD=${arr[2]} # distributed pre-release build conda mambabuild continuous_integration/recipes/distributed \ From 75af34e959187c9287387f0cb60c0502bd14b631 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 12 Jan 2022 11:22:07 -0500 Subject: [PATCH 09/11] Add conda build config --- .../recipes/distributed/conda_build_config.yaml | 5 +++++ continuous_integration/recipes/distributed/meta.yaml | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 continuous_integration/recipes/distributed/conda_build_config.yaml diff --git a/continuous_integration/recipes/distributed/conda_build_config.yaml b/continuous_integration/recipes/distributed/conda_build_config.yaml new file mode 100644 index 00000000000..c914f7d2a20 --- /dev/null +++ b/continuous_integration/recipes/distributed/conda_build_config.yaml @@ -0,0 +1,5 @@ +python: + - 3.7 + - 3.8 + - 3.9 + - 3.10 \ No newline at end of file diff --git a/continuous_integration/recipes/distributed/meta.yaml b/continuous_integration/recipes/distributed/meta.yaml index b86e5f1a85f..c998c435d44 100644 --- a/continuous_integration/recipes/distributed/meta.yaml +++ b/continuous_integration/recipes/distributed/meta.yaml @@ -1,7 +1,6 @@ {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} {% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %} {% set dask_build = environ.get('DASK_BUILD', '') %} -{% set py_version=environ.get('CONDA_PY', 36) %} package: @@ -13,7 +12,7 @@ source: build: number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script: {{ PYTHON }} -m pip install . -vv entry_points: - dask-scheduler = distributed.cli.dask_scheduler:go From 004813c709802fb7289d004113c1028b000e2863 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:47:46 -0500 Subject: [PATCH 10/11] Add newline to build config --- .../recipes/distributed/conda_build_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/recipes/distributed/conda_build_config.yaml b/continuous_integration/recipes/distributed/conda_build_config.yaml index c914f7d2a20..c5c49ce7a06 100644 --- a/continuous_integration/recipes/distributed/conda_build_config.yaml +++ b/continuous_integration/recipes/distributed/conda_build_config.yaml @@ -2,4 +2,4 @@ python: - 3.7 - 3.8 - 3.9 - - 3.10 \ No newline at end of file + - 3.10 From 4d016820454c2296f3c1c8110863483b3029924c Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 14 Jan 2022 10:07:25 -0500 Subject: [PATCH 11/11] Don't build Python 3.10 nightlies --- .../recipes/distributed/conda_build_config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/continuous_integration/recipes/distributed/conda_build_config.yaml b/continuous_integration/recipes/distributed/conda_build_config.yaml index c5c49ce7a06..378349abc92 100644 --- a/continuous_integration/recipes/distributed/conda_build_config.yaml +++ b/continuous_integration/recipes/distributed/conda_build_config.yaml @@ -2,4 +2,3 @@ python: - 3.7 - 3.8 - 3.9 - - 3.10