Merge pull request #1 from lenskit/feature/build-update

Update build logic & metadata

.github/workflows/test.yml

@@ -27,30 +27,31 @@ jobs:
         fetch-depth: 0
         submodules: true
 
-    - name: Setup conda
-      uses: s-weigand/setup-conda@v1
-      with:
-        update-conda: true
-        python-version: ${{ matrix.python }}
-        conda-channels: conda-forge
-
-    - name: Set up dependencies
+    - name: Prepare Conda environment spec
+      id: conda-lock
+      shell: bash
       run: |
-        conda install -qy flit-core packaging pyyaml
-        python build-tools/flit-conda.py -E mkl --python-version ${{matrix.python}} --save-env environment.yml
-        cat environment.yml
-        conda env create -q -n lk-hpf -f environment.yml
+        bash lkbuild/lock-for-ci.sh -v $PYVER
+      env:
+        RUNNER: ${{runner.os}}
+        PYVER: ${{matrix.python}}
+
+    - name: Initialize Conda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: lk-hpf
+        environment-file: ${{steps.conda-lock.outputs.environment-file}}
 
     - name: 🔎 Inspect environment
       run: |
-        conda run -n lk-hpf numba -s
+        numba -s
 
     - name: Run tests
       run: |
-        conda run -n lk-hpf --no-capture-output python -m pytest --cov=lenskit_hpf -v
+        python -m pytest --cov=lenskit_hpf -v
 
     - name: Aggregate coverage data
-      run: conda run -n lk-hpf coverage xml
+      run: coverage xml
 
     - name: Upload coverage
       uses: codecov/codecov-action@v1

lkbuild/boot-env.yml

@@ -0,0 +1,11 @@
+name: lkboot
+channels:
+- conda-forge
+dependencies:
+- python>=3.8
+- flit=3
+- invoke=1
+- requests=2
+- pip
+- conda-lock
+- mamba

lkbuild/bootstrap-env.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Script to bootstrap an environment for running the infra code.
+# The resulting environment can be used to run 'invoke' and get a
+# Conda lock for a full development environment.
+#
+# This script is intended to be sourced, but can be run on its own.
+
+msg()
+{
+    echo "$@" >&2
+}
+
+err()
+{
+    echo "ERR:" "$@" >&2
+    exit 3
+}
+
+vr()
+{
+  local ec
+  echo + "$@" >&2
+  "$@"
+  ec="$?"
+  if [ $ec -ne 0 ]; then
+      echo "command $1 terminated with code $ec" >&2
+      exit 2
+  fi
+}
+
+setup_micromamba()
+{
+  msg "Installing Micromamba"
+  CONDA_PLATFORM=$(python3 lkbuild/env.py)
+  test "$?" -eq 0 || exit 2
+  mkdir -p build/mmboot
+  vr wget -qO build/micromamba.tar.bz2 https://micromamba.snakepit.net/api/micromamba/$CONDA_PLATFORM/latest
+  vr tar -C build/mmboot -xvjf build/micromamba.tar.bz2
+  MM=build/mmboot/bin/micromamba
+  if [ ! -e "$MM" ]; then
+    # this should only happen on Windows
+    MM=build/mmboot/Library/bin/micromamba.exe
+  fi
+  eval "$($MM shell hook -p $HOME/micromamba -s bash)"
+}
+
+setup_boot_env()
+{
+  msg "Installing bootstrap environment"
+  vr micromamba env create -qy -n lkboot -f lkbuild/boot-env.yml
+  msg "Activating bootstrap environment"
+  micromamba activate lkboot
+}
+
+setup_micromamba
+setup_boot_env
+
+msg "Inspecting environment"
+CONDA_PLATFORM=$(invoke conda-platform)

lkbuild/env.py

@@ -0,0 +1,22 @@
+import sysconfig
+import re
+
+
+def conda_platform():
+    plat = sysconfig.get_platform()
+    if re.match(r'^macosx-.*-x86_64', plat):
+        return 'osx-64'
+    if re.match(r'^macosx-.*-arm64', plat):
+        return 'osx-arm64'
+    if re.match(r'^[Ll]inux.*-x86_64', plat):
+        return 'linux-64'
+    if re.match(r'^[Ll]inux.*-aarch64', plat):
+        return 'linux-aarch64'
+    if plat == 'win-amd64':
+        return 'win-64'
+
+    raise ValueError('unrecognized platform ' + plat)
+
+
+if __name__ == '__main__':
+    print(conda_platform())

lkbuild/lock-for-ci.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Create a lock file for CI builds, and register with GitHub outputs.
+
+SDIR=$(dirname $0)
+
+. "$SDIR/bootstrap-env.sh"
+
+vr invoke dev-lock "$@"
+echo "::set-output name=environment-file::conda-$CONDA_PLATFORM.lock"

lkbuild/mkl-spec.yml

@@ -0,0 +1,4 @@
+# environment mini-spec for ensuring MKL
+dependencies:
+  - libblas=*=*mkl
+  - mkl=2020

lkbuild/openblas-spec.yml

@@ -0,0 +1,3 @@
+# environment mini-spec for ensuring OpenBLAS
+dependencies:
+  - libblas=*=*openblas

lkbuild/python-3.10-spec.yml

@@ -0,0 +1,3 @@
+# environment mini-spec for ensuring Python 3.10
+dependencies:
+- python=3.10

lkbuild/python-3.7-spec.yml

@@ -0,0 +1,3 @@
+# environment mini-spec for ensuring Python 3.7
+dependencies:
+- python=3.7

lkbuild/python-3.8-spec.yml

@@ -0,0 +1,3 @@
+# environment mini-spec for ensuring Python 3.8
+dependencies:
+- python=3.8

lkbuild/python-3.9-spec.yml

@@ -0,0 +1,3 @@
+# environment mini-spec for ensuring Python 3.9
+dependencies:
+- python=3.9

lkbuild/tasks.py

@@ -0,0 +1,48 @@
+"""
+Support tasks shared across LensKit packages.
+"""
+
+import sys
+from invoke import task
+from . import env
+
+__ALL__ = [
+    'dev_lock',
+    'conda_platform'
+]
+
+
+@task(iterable=['extras'])
+def dev_lock(c, platform=None, extras=None, version=None, blas=None, env_file=False):
+    "Create a development lockfile"
+    plat = env.conda_platform()
+
+    if platform == 'all':
+        plat_opt = ''
+    elif platform:
+        plat_opt = f'-p {platform}'
+    else:
+        plat_opt = f'-p {plat}'
+
+    cmd = f'conda-lock lock --mamba {plat_opt} -f pyproject.toml'
+    if env_file:
+        cmd += ' -k env'
+
+    if version:
+        cmd += f' -f lkbuild/python-{version}-spec.yml'
+    if blas:
+        cmd += f' -f lkbuild/{blas}-spec.yml'
+    for e in extras:
+        cmd += f' -e {e}'
+
+    print('running', cmd, file=sys.stderr)
+    c.run(cmd)
+
+
+@task
+def conda_platform(c, gh_output=False):
+    plat = env.conda_platform()
+    if gh_output:
+        print('::set-output name=conda-platform::' + plat)
+    else:
+        print(plat)

pyproject.toml

@@ -1,13 +1,12 @@
 [build-system]
-requires = ["flit_core >=2,<4"]
+requires = ["flit_core >=3.2,<4"]
 build-backend = "flit_core.buildapi"
 
-[tool.flit.metadata]
-module = "lenskit_hpf"
-dist-name = "lenskit-hpf"
-author = "Michael Ekstrand"
-author-email = "michaelekstrand@boisestate.edu"
-home-page = "https://hpf.lenskit.org"
+[project]
+name = "lenskit-hpf"
+authors = [
+    { name = "Michael Ekstrand", email = "michaelekstrand@boisestate.edu" }
+]
 classifiers = [
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
@@ -17,19 +16,20 @@ classifiers = [
     "Operating System :: OS Independent",
     "Intended Audience :: Science/Research",
 ]
+readme = "README.md"
+license = { file="LICENSE.md" }
 requires-python = ">= 3.7"
-description-file = "README.md"
-license = "MIT"
-requires = [
+dependencies = [
     "lenskit[doc] @ git+https://github.com/lenskit/lkpy.git#main",
     "hpfrec"
 ]
+dynamic = ["description", "version"]
 
-[tool.flit.metadata.urls]
-Documentation = "https://lkpy.lenskit.org/en/stable/hpfrec"
+[project.urls]
+homepage = "https://lkpy.lenskit.org/projects/lenskit-hpf"
 GitHub = "https://github.com/lenskit/lenskit-hpf"
 
-[tool.flit.metadata.requires-extra]
+[project.optional-dependencies]
 test = [
     "pytest >= 6",
     "pytest-doctestplus >= 0.9",
@@ -42,31 +42,19 @@ dev = [
     "pytest-cov >= 2.12",
 ]
 
+[tool.flit.module]
+name = "lenskit_hpf"
+
 [tool.flit.sdist]
 exclude = [
     ".github",
     "*.ipynb",
-    "doc",
+    "docs",
     "data",
     "examples",
     "build-tools",
     "tests",
 ]
 
-[tool.envtool.conda]
-name = "lkpy"
-channels = [
-    "conda-forge"
-]
-
-[tool.envtool.conda.extras]
-".none" = [
-    "numba =0.53",
-    "pandas =1.2",
-    "csr >=0.4",
-    "cython",
-    "scipy",
-]
-
-[tool.envtool.conda.overrides]
-hpfrec = { source = "pip" }
+[tool.conda-lock]
+channels = ["conda-forge"]

tasks.py

@@ -0,0 +1 @@
+from lkbuild.tasks import *