diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 00000000000..d4f04ff6a6b --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,81 @@ +name: Conda build +on: + push: + branches: + - main + pull_request: + +# When this workflow is queued, automatically cancel any previous running +# or pending jobs from the same branch +concurrency: + group: conda-${{ github.head_ref }} + cancel-in-progress: true + +# Required shell entrypoint to have properly activated conda environments +defaults: + run: + shell: bash -l {0} + +jobs: + conda: + name: Build (and upload) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + use-mamba: true + python-version: 3.8 + - name: Install dependencies + run: | + mamba install boa conda-verify + + which python + pip list + mamba list + - name: Build conda packages + run: | + # suffix for pre-release package versions + export VERSION_SUFFIX=a`date +%y%m%d` + + # conda search for the latest dask-core pre-release + arr=($(conda search --override-channels -c dask/label/dev dask-core | tail -n 1)) + + # extract dask-core pre-release version / build + export DASK_VERSION=${arr[1]} + export DASK_BUILD=${arr[2]} + + # distributed pre-release build + conda mambabuild continuous_integration/recipes/distributed \ + --channel dask/label/dev \ + --no-anaconda-upload \ + --output-folder build + + # dask pre-release build + conda mambabuild continuous_integration/recipes/dask \ + --channel dask/label/dev \ + --no-anaconda-upload \ + --output-folder build + - name: Upload conda packages + if: | + github.event_name == 'push' + && github.ref == 'refs/heads/main' + && github.repository == 'dask/distributed' + env: + ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }} + run: | + # convert distributed to other architectures + cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \ + -p osx-arm64 \ + -p linux-ppc64le \ + -p linux-aarch64 \ + -p win-64 + + # install anaconda for upload + mamba install anaconda-client + + anaconda upload --label dev **.tar.bz2 diff --git a/continuous_integration/recipes/dask/meta.yaml b/continuous_integration/recipes/dask/meta.yaml new file mode 100644 index 00000000000..10d81f5bbd8 --- /dev/null +++ b/continuous_integration/recipes/dask/meta.yaml @@ -0,0 +1,56 @@ +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %} +{% set dask_build = environ.get('DASK_BUILD', '') %} +{% set py_version=environ.get('CONDA_PY', 36) %} + + +package: + name: dask + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + noarch: python + +requirements: + host: + - python >=3.7 + run: + - python >=3.7 + - dask-core {{ dask_version }}={{ dask_build }} + - distributed {{ version }}=py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + - cytoolz >=0.8.2 + - numpy >=1.18 + - pandas >=1.0 + - bokeh >=2.1.1 + - jinja2 + + run_constrained: + - openssl !=1.1.1e + +test: + imports: + - dask + - dask.array + - dask.bag + - dask.bytes + - dask.dataframe + - dask.dataframe.tseries + - dask.delayed + - dask.diagnostics + - dask.distributed + - distributed + +about: + home: https://dask.org/ + summary: Parallel PyData with Task Scheduling + license: BSD-3-Clause + license_file: LICENSE.txt + description: | + Dask is a flexible parallel computing library for analytics. + doc_url: https://dask.org/ + dev_url: https://github.com/dask/dask diff --git a/continuous_integration/recipes/distributed/conda_build_config.yaml b/continuous_integration/recipes/distributed/conda_build_config.yaml new file mode 100644 index 00000000000..378349abc92 --- /dev/null +++ b/continuous_integration/recipes/distributed/conda_build_config.yaml @@ -0,0 +1,4 @@ +python: + - 3.7 + - 3.8 + - 3.9 diff --git a/continuous_integration/recipes/distributed/meta.yaml b/continuous_integration/recipes/distributed/meta.yaml new file mode 100644 index 00000000000..c998c435d44 --- /dev/null +++ b/continuous_integration/recipes/distributed/meta.yaml @@ -0,0 +1,79 @@ +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev') + environ.get('VERSION_SUFFIX', '') %} +{% set dask_version = environ.get('DASK_VERSION', '0.0.0.dev') %} +{% set dask_build = environ.get('DASK_BUILD', '') %} + + +package: + name: distributed + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script: {{ PYTHON }} -m pip install . -vv + entry_points: + - dask-scheduler = distributed.cli.dask_scheduler:go + - dask-ssh = distributed.cli.dask_ssh:go + - dask-worker = distributed.cli.dask_worker:go + +requirements: + build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + host: + - python + - pip + + run: + - python + - click >=6.6 + - cloudpickle >=1.5.0 + - cytoolz >=0.8.2 + - dask-core {{ dask_version }}={{ dask_build }} + - jinja2 + - msgpack-python >=0.6.0 + - psutil >=5.0 + - pyyaml + - sortedcontainers !=2.0.0,!=2.0.1 + - tblib >=1.6.0 + - toolz >=0.8.2 + - tornado >=5 # [py<38] + - tornado >=6.0.3 # [py>=38] + - zict >=0.1.3 + - setuptools + + run_constrained: + - openssl !=1.1.1e + +test: + imports: + - distributed + - distributed.cli + - distributed.comm + - distributed.deploy + - distributed.diagnostics + - distributed.protocol + commands: + - pip check + - dask-scheduler --help + - dask-ssh --help + - dask-worker --help + requires: + - pip + +about: + home: https://distributed.dask.org + summary: Distributed scheduler for Dask + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE.txt + description: | + Distributed is a lightweight library for distributed computing in Python. + It extends both the concurrent.futures and dask APIs to moderate sized + clusters. + doc_url: https://distributed.dask.org + doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst + dev_url: https://github.com/dask/distributed