Add workflow / recipe to generate Dask/distributed pre-releases

.github/workflows/conda.yml

@@ -0,0 +1,81 @@
+name: Conda build
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs from the same branch
+concurrency:
+  group: conda-${{ github.head_ref }}
+  cancel-in-progress: true
+
+# Required shell entrypoint to have properly activated conda environments
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  conda:
+    name: Build (and upload)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          mamba install boa conda-verify
+
+          which python
+          pip list
+          mamba list
+      - name: Build conda packages
+        run: |
+          # suffix for pre-release package versions
+          export VERSION_SUFFIX=a`date +%y%m%d`
+
+          # conda search for the latest dask-core pre-release
+          arr=($(conda search --override-channels -c dask/label/dev dask-core | tail -n 1))
+
+          # extract dask-core pre-release version / build
+          export DASK_VERSION=${arr[1]}
+          export DASK_BUILD=${arr[2]}
+
+          # distributed pre-release build
+          conda mambabuild continuous_integration/recipes/distributed \
+                           --channel dask/label/dev \
+                           --no-anaconda-upload \
+                           --output-folder build
+
+          # dask pre-release build
+          conda mambabuild continuous_integration/recipes/dask \
+                           --channel dask/label/dev \
+                           --no-anaconda-upload \
+                           --output-folder build
+      - name: Upload conda packages
+        if: |
+          github.event_name == 'push'
+          && github.ref == 'refs/heads/main'
+          && github.repository == 'dask/distributed'
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
+        run: |
+          # convert distributed to other architectures
+          cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \
+                                                       -p osx-arm64 \
+                                                       -p linux-ppc64le \
+                                                       -p linux-aarch64 \
+                                                       -p win-64
+
+          # install anaconda for upload
+          mamba install anaconda-client
+
+          anaconda upload --label dev **.tar.bz2

continuous_integration/recipes/dask/meta.yaml

@@ -0,0 +1,56 @@
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %}
+{% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %}
+{% set dask_build = environ.get('DASK_BUILD', '') %}
+{% set py_version=environ.get('CONDA_PY', 36) %}
+
+
+package:
+  name: dask
+  version: {{ version }}
+
+source:
+  git_url: ../../..
+
+build:
+  number: {{ GIT_DESCRIBE_NUMBER }}
+  string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  noarch: python
+
+requirements:
+  host:
+    - python >=3.7
+  run:
+    - python >=3.7
+    - dask-core {{ dask_version }}={{ dask_build }}
+    - distributed {{ version }}=py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    - cytoolz >=0.8.2
+    - numpy >=1.18
+    - pandas >=1.0
+    - bokeh >=2.1.1
+    - jinja2
+
+  run_constrained:
+    - openssl !=1.1.1e
+
+test:
+  imports:
+    - dask
+    - dask.array
+    - dask.bag
+    - dask.bytes
+    - dask.dataframe
+    - dask.dataframe.tseries
+    - dask.delayed
+    - dask.diagnostics
+    - dask.distributed
+    - distributed
+
+about:
+  home: https://dask.org/
+  summary: Parallel PyData with Task Scheduling
+  license: BSD-3-Clause
+  license_file: LICENSE.txt
+  description: |
+    Dask is a flexible parallel computing library for analytics.
+  doc_url: https://dask.org/
+  dev_url: https://github.com/dask/dask

continuous_integration/recipes/distributed/conda_build_config.yaml

@@ -0,0 +1,4 @@
+python:
+    - 3.7
+    - 3.8
+    - 3.9

continuous_integration/recipes/distributed/meta.yaml

@@ -0,0 +1,79 @@
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %}
+{% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %}
+{% set dask_build = environ.get('DASK_BUILD', '') %}
+
+
+package:
+  name: distributed
+  version: {{ version }}
+
+source:
+  git_url: ../../..
+
+build:
+  number: {{ GIT_DESCRIBE_NUMBER }}
+  string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  script: {{ PYTHON }} -m pip install . -vv
+  entry_points:
+    - dask-scheduler = distributed.cli.dask_scheduler:go
+    - dask-ssh = distributed.cli.dask_ssh:go
+    - dask-worker = distributed.cli.dask_worker:go
+
+requirements:
+  build:
+    - python                                 # [build_platform != target_platform]
+    - cross-python_{{ target_platform }}     # [build_platform != target_platform]
+  host:
+    - python
+    - pip
+
+  run:
+    - python
+    - click >=6.6
+    - cloudpickle >=1.5.0
+    - cytoolz >=0.8.2
+    - dask-core {{ dask_version }}={{ dask_build }}
+    - jinja2
+    - msgpack-python >=0.6.0
+    - psutil >=5.0
+    - pyyaml
+    - sortedcontainers !=2.0.0,!=2.0.1
+    - tblib >=1.6.0
+    - toolz >=0.8.2
+    - tornado >=5  # [py<38]
+    - tornado >=6.0.3  # [py>=38]
+    - zict >=0.1.3
+    - setuptools
+
+  run_constrained:
+    - openssl !=1.1.1e
+
+test:
+  imports:
+    - distributed
+    - distributed.cli
+    - distributed.comm
+    - distributed.deploy
+    - distributed.diagnostics
+    - distributed.protocol
+  commands:
+    - pip check
+    - dask-scheduler --help
+    - dask-ssh --help
+    - dask-worker --help
+  requires:
+    - pip
+
+about:
+  home: https://distributed.dask.org
+  summary: Distributed scheduler for Dask
+  license: BSD-3-Clause
+  license_family: BSD
+  license_file: LICENSE.txt
+  description: |
+    Distributed is a lightweight library for distributed computing in Python.
+    It extends both the concurrent.futures and dask APIs to moderate sized
+    clusters.
+  doc_url: https://distributed.dask.org
+  doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst
+  dev_url: https://github.com/dask/distributed