Add/spack yaml support (#6)

* test environment package * update readme Signed-off-by: vsoch <vsoch@users.noreply.github.com> Co-authored-by: vsoch <vsoch@users.noreply.github.com>
vsoch · Dec 1, 2021 · 2f0a935 · 2f0a935
1 parent ef447c8
commit 2f0a935
Show file tree

Hide file tree

Showing 6 changed files with 328 additions and 2 deletions.
diff --git a/.github/workflows/test-action.yaml b/.github/workflows/test-action.yaml
@@ -36,7 +36,22 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           deploy: false
-
+
+  build-env-binaries:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    name: Build Environment Binaries
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build Environment
+        uses: ./envpackage
+        with:
+          spack_yaml: spack/spack.yaml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          deploy: ${{ github.event_name != 'pull_request' }}
+
   build-container-spack-yaml:
     runs-on: ubuntu-latest
     permissions:

diff --git a/README.md b/README.md
@@ -85,14 +85,14 @@ jobs:
         uses: vsoch/spack-package-action/package@main
         with:
           package: zlib
-          deploy: true
           token: ${{ secrets.GITHUB_TOKEN }}
           deploy: ${{ github.event_name != 'pull_request' }}
 ```
 
 This action is provided in [package](package), and an example package is shown 
 [here](https://github.com/vsoch/spack-package-action/pkgs/container/spack-package-action%2Fzlib).
 
+
 ### Oras Pull
 
 To then get the binary, you can [install oras](https://oras.land/cli/) and do:
@@ -133,6 +133,47 @@ spack build hashes (that identify the package) are going to vary. This could be
 of different supported packages, or bad if you want consistently the same one. Likely you can pin this by setting a target
 in the `flags` for the package.
 
+## Environment Binary Build
+
+The environment builder is similar to the package builder, but instead of a package name
+you provide a path to a spack.yaml, and the entire environment is built and added to the build
+cache. This can be useful for a package to provide it's own buildcache. Here is an example:
+
+
+```yaml
+jobs:
+  build-env-binaries:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    name: Build Environment Binaries
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build Environment
+        uses: vsoch/spack-package-action/envpackage@main
+        with:
+          spack_yaml: spack/spack.yaml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          deploy: ${{ github.event_name != 'pull_request' }}
+```
+
+This action is provided in [envpackage](envpackage).
+
+### Variables
+
+| name | description | default | example | required |
+|------|-------------|---------|---------|----------|
+| spack_yaml | path to spack.yaml to generate environment | unset | spack/spack.yaml | true |
+| repos | comma separated list of additional repos to clone and add | unset | https://github.com/rbogle/spack-repo | false
+| branch | The branch of spack to use | develop | feature-branch | false | 
+| release | A spack release to use (if defined, overrides branch) | unset | 0.17.0 | false |
+| token | A GitHub token required if releasing artifacts to the same repository | unset | `${{ secrets.GITHUB_TOKEN }}` | false |
+| flags | Extra flags (compiler, target, variants, etc) to add to the install command | unset | +debug | false |
+| tag | Tag to use for package | latest | v10.0.0 | false |
+| deploy | Deploy (release) package to GitHub repository (token is required) | false | true | true |
+
+
 ## Package Container Build
 
 If you instead want to provide a container for your package, you can do that too!

diff --git a/envpackage/action.yml b/envpackage/action.yml
@@ -0,0 +1,97 @@
+name: "Spack Binary Release Action"
+description: "Build and release a package.py in a source repository with spack."
+inputs:
+  spack_yaml:
+    description: A spack yaml to install and add to the build cache
+    required: true
+  repos:
+    description: comma separated list of additional repos to clone and add
+    required: false
+  branch:
+    description: Spack branch to clone
+    required: false
+    default: develop
+  release:
+    description: A spack release to use (if defined, overrides branch)
+    required: false
+    default: ""
+  token:
+    description: A GitHub token required if releasing artifacts to the same repository
+    required: true
+  flags:
+    description: Extra flags (compiler, target, variants, etc) to add to the install command
+    required: false
+  tag:
+    description: Tag to use for package (defaults to latest)
+    required: false
+  deploy:
+    description: Deploy (release) package to GitHub repository (token is required)
+    required: true
+    default: false
+
+outputs:
+  package_name:
+    description: "package name uploaded to"
+    value: ${{ steps.release.outputs.package_name }}
+  package_tagged_name:
+    description: "package name with custom tag uploaded to"
+    value: ${{ steps.release.outputs.package_name }}
+  package_tag:
+    description: "package tag (along with latest)"
+    value: ${{ steps.release.outputs.package_tag }}
+  package_content_type:
+    description: "the package content type"
+    value: ${{ steps.release.outputs.content_type }}
+  spec_jsons:
+    description: "comma separately list of json files in the cache to describe the entry."
+    value: ${{ steps.build.outputs.spec_jsons }}
+  build_cache:
+    description: "Build cache path (dated YY.MM)"
+    value: ${{ steps.build.outputs.build_cache }}
+  build_cache_prefix:
+    description: Prefix of spack build cache (directory above build_cache)
+    value: ${{ steps.build.outputs.build_cache_prefix }}
+  spec:
+    description: "Spec used for build"
+    value: ${{ steps.build.outputs.spec }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set Root Directory
+      env:
+        ACTION_PATH: ${{ github.action_path }}
+      run: ${{ github.action_path }}/scripts/set_root.sh
+      shell: bash
+
+    - name: Install Spack and Dependencies
+      env:
+        INPUT_SPACK_BRANCH: ${{ inputs.branch }}
+        INPUT_RELEASE: ${{ inputs.release }}
+        INPUT_REPOS: ${{ inputs.repos }}
+        INPUT_SPACK_ROOT: /opt/spack
+      run: ${{ env.ACTION_ROOT }}/install/scripts/install.sh
+      shell: bash
+
+    - name: Build Packages
+      id: build
+      env:
+        INPUT_SPACK_YAML: ${{ inputs.spack_yaml }}
+        INPUT_FLAGS: ${{ inputs.flags }}
+        ACTION_ROOT: ${{ github.action_path }}
+      run: ${{ github.action_path }}/scripts/build.sh
+      shell: bash
+
+    - name: Package Release
+      id: release
+      env:
+        INPUT_PACKAGE_NAME: ${{ inputs.package }}
+        GITHUB_TOKEN: ${{ inputs.token }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        INPUT_TAG: ${{ inputs.tag }}
+        BUILD_CACHE_PREFIX: ${{ env.build_cache_prefix }}
+        BUILD_CACHE: ${{ env.build_cache }}
+        SPACK_ROOT: /opt/spack
+        DEPLOY: ${{ inputs.deploy }}
+      run: ${{ github.action_path }}/scripts/release.sh
+      shell: bash
diff --git a/envpackage/scripts/build.sh b/envpackage/scripts/build.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -e
+
+echo $PWD
+ls 
+
+
+# Setup the spack environment
+. /opt/spack/share/spack/setup-env.sh 
+
+# The spack yaml must exist
+if [ ! -f "${INPUT_SPACK_YAML}" ]; then
+    printf "${INPUT_SPACK_YAML} does not exist\n"
+    exit
+fi
+
+# Show user all variables for debugging
+printf "spack_yaml: ${INPUT_SPACK_YAML}\n"
+printf "flags: ${INPUT_FLAGS}\n"
+printf "spec: ${SPACK_SPEC}\n"
+
+# Do we want a custom compiler / variants?
+if [ ! -z ${INPUT_FLAGS} ]; then
+    SPACK_SPEC="$SPACK_SPEC ${INPUT_FLAGS}"
+fi
+
+# Create the spack environment
+envdirname=$(dirname ${INPUT_SPACK_YAML})
+cd $envdirname
+
+# And install packages to it
+spack env create -d .
+spack env activate .
+spack install
+
+# After install, create and add to build cache.
+# We want the directory to be the YEAR.MONTH (21.05)
+month=$(date '+%y.%m')
+build_cache=/opt/${month}
+mkdir -p $build_cache
+
+# Add the key, stored with buildcache action (we need to do both these things?)
+root=$(dirname ${ACTION_ROOT})
+spack gpg trust ${root}/buildcache/4A424030614ADE118389C2FD27BDB3E5F0331921.pub
+spack gpg init
+spack gpg create "${GITHUB_ACTOR}" "${GITHUB_ACTOR}@users.noreply.github.com"
+
+# Install packages. If needed, we can add a variable to customize the string here
+spack buildcache create -a -d  ${build_cache} $(spack find --format "{name}/{hash}")
+
+# Did we make stuff?
+tree ${build_cache}
+
+# We want to save the .json files for any following step :)
+spec_jsons=""
+
+echo "::set-output name=build_cache::${build_cache}"
+echo "::set-output name=build_cache_prefix::${build_cache_prefix}"
+
+# There can be more than one thing in the build cache
+for spec_json in $(find ${build_cache} -name *.json); do
+    printf "${spec_json}\n"
+    cat ${spec_json}
+    if [[ "${spec_jsons}" == "" ]]; then
+        spec_jsons=${spec_json}
+    else
+        spec_jsons="${spec_jsons},${spec_json}"
+    fi
+done
+
+# Set output for spec, and TODO binary to upload/save for next step
+echo "::set-output name=spec::${SPACK_SPEC}"
+echo "::set-output name=spec_jsons::${spec_jsons}"
+echo "spec=${SPACK_SPEC}" >> $GITHUB_ENV
+echo "spec_jsons=${spec_jsons}" >> $GITHUB_ENV
+echo "build_cache=${build_cache}" >> $GITHUB_ENV
+echo "build_cache_prefix=${month}/build_cache" >> $GITHUB_ENV
diff --git a/envpackage/scripts/release.sh b/envpackage/scripts/release.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -e
+
+echo $PWD
+ls 
+
+printf "build cache prefix: ${BUILD_CACHE_PREFIX}"
+printf "build cache: ${BUILD_CACHE}"
+printf "package: ${INPUT_PACKAGE_NAME}\n"
+printf "actor: ${GITHUB_ACTOR}\n"
+printf "tag: ${INPUT_TAG}\n"
+printf "deploy: ${DEPLOY}\n"
+
+# Setup the spack environment
+. "${SPACK_ROOT}/share/spack/setup-env.sh"
+
+# Install oras
+curl -LO https://github.com/oras-project/oras/releases/download/v0.12.0/oras_0.12.0_linux_amd64.tar.gz
+mkdir -p oras-install/
+tar -zxf oras_0.12.0_*.tar.gz -C oras-install/
+mv oras-install/oras /usr/local/bin/
+rm -rf oras_0.12.0_*.tar.gz oras-install/
+
+# Login to GitHub packages
+if [ "${DEPLOY}" == "true" ]; then
+    echo ${GITHUB_TOKEN} | oras login -u ${GITHUB_ACTOR} --password-stdin ghcr.io
+fi
+
+# Do we have a tag?
+if [ -z "${INPUT_TAG}" ]; then
+    INPUT_TAG=${GITHUB_SHA:0:8}
+fi
+
+# Package content type is always consistent
+package_content_type=application/vnd.spack.package
+echo "::set-output name=package_content_type::${package_content_type}"
+echo "package_content_type=${package_content_type}" >> $GITHUB_ENV
+
+# As is common input tag
+echo "::set-output name=package_tag::${INPUT_TAG}"
+echo "package_tag=${INPUT_TAG}" >> $GITHUB_ENV
+
+# Keep lists of package names, tagged, names
+tagged_names=""
+package_names=""
+
+# The package name must include the package and hash, etc.
+# linux-ubuntu20.04-broadwell-gcc-10.3.0-zlib-1.2.11-5vlodp7yawk5elx4dfhnpzmpg743fwv3.spack
+for spack_package in $(find ${BUILD_CACHE} -name *.spack); do
+
+    spack_package_name=$(basename $spack_package)
+    package_name="${build_cache_prefix}/${spack_package_name}"
+
+    # Absolute paths not allowed
+    mv ${spack_package} ${spack_package_name}
+
+    package_full_name=ghcr.io/${GITHUB_REPOSITORY}/${package_name}:latest
+    package_tagged_name=ghcr.io/${GITHUB_REPOSITORY}/${package_name}:${INPUT_TAG}
+
+    # Add to listing, comma separated
+    if [ "${tagged_names}" == "" ]; then
+        tagged_names="${package_tagged_name}"    
+        package_names="${package_full_name}" 
+    else
+        tagged_names="${tagged_names},${package_tagged_name}"    
+        package_names="${package_names},${package_full_name}"     
+    fi
+
+    # Push for latest
+    if [ "${DEPLOY}" == "true" ]; then
+        printf "oras push ${package_full_name} --manifest-config /dev/null:${package_content_type} ${spack_package_name}\n"
+        oras push ${package_full_name} --manifest-config /dev/null:${package_content_type} ${spack_package_name}
+
+        # And custom tag (which will default to GITHUB_SHA)
+        printf "oras push ${package_tagged_name} --manifest-config /dev/null:${package_content_type} ${spack_package_name}\n"
+        oras push  ${package_tagged_name} --manifest-config /dev/null:${package_content_type} ${spack_package_name}
+    fi
+done
+
+echo "::set-output name=package_name::${package_names}"
+echo "::set-output name=package_tagged_names::${tagged_names}"
+echo "package_names=${package_names}" >> $GITHUB_ENV
+echo "package_tagged_names=${tagged_names}" >> $GITHUB_ENV
diff --git a/envpackage/scripts/set_root.sh b/envpackage/scripts/set_root.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+echo $PWD
+ls 
+
+ACTION_ROOT=$(dirname $ACTION_PATH)
+printf "ACTION_ROOT is ${ACTION_ROOT}\n"
+ls ${ACTION_ROOT}
+echo "ACTION_ROOT=${ACTION_ROOT}" >> $GITHUB_ENV