Alma9 build v5 nightly release #257
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Alma9 build v5 nightly release | |
on: | |
workflow_dispatch: | |
inputs: | |
tag-prefix: | |
description: 'nightly tag prefix' | |
default: '' | |
feature-branch: | |
description: 'feature branch to be used across all DAQ repos wherever possible.' | |
default: develop | |
cvmfs-deployment: | |
description: 'whether to deploy the release to cvmfs' | |
default: 'no' | |
env: | |
EXTERNALS_IMAGE: alma9-slim-externals:v2.1pt | |
EXTERNALS_TAG: v2.1pt | |
USERNAME: dune-daq | |
REGISTRY: ghcr.io | |
#ACTION_ID: ${{ github.run_id }}-${{ github.run_attempt }} | |
jobs: | |
make_nightly_tag: | |
name: create nightly tag | |
runs-on: ubuntu-latest | |
outputs: | |
tag: ${{ steps.create_nightly_tag.outputs.nightly_tag }} | |
defaults: | |
run: | |
shell: bash | |
steps: | |
- id: create_nightly_tag | |
run: | | |
date_tag=$(date +%y%m%d) | |
echo "nightly_tag=${{ github.event.inputs.tag-prefix }}_DEV_${date_tag}_A9" >> "$GITHUB_OUTPUT" | |
cat "$GITHUB_OUTPUT" | |
build_the_develop_release_spack: | |
name: build_dev_release_spack | |
runs-on: daq | |
needs: make_nightly_tag | |
container: | |
image: ghcr.io/dune-daq/alma9-slim-externals:v2.1pt | |
defaults: | |
run: | |
shell: bash | |
steps: | |
- name: Checkout daq-release | |
uses: actions/checkout@v4 | |
with: | |
path: daq-release | |
- name: setup directories and install spack for the coredaq release | |
env: | |
NIGHTLY_TAG: ${{needs.make_nightly_tag.outputs.tag}} | |
run: | | |
export BASE_RELEASE_DIR=/cvmfs/dunedaq-development.opensciencegrid.org/nightly/NB${NIGHTLY_TAG} | |
export DET_RELEASE_DIR=/cvmfs/dunedaq-development.opensciencegrid.org/nightly/NFD${NIGHTLY_TAG} | |
export OS=almalinux9 | |
source daq-release/.github/workflows/wf-setup-tools.sh | |
daq-release/scripts/spack/build-release.sh $BASE_RELEASE_DIR $DET_RELEASE_DIR core $OS ${{ github.event.inputs.feature-branch }} | |
cd $BASE_RELEASE_DIR/../ | |
tar_and_stage_release ${BASE_RELEASE_TAG} | |
- name: upload spack nightly tarball for base release | |
uses: actions/upload-artifact@v3 | |
with: | |
name: nightly_coredaq | |
path: ${{ github.workspace }}/tarballs_for_upload/NB${{needs.make_nightly_tag.outputs.tag}}.tar.gz | |
- name: setup directories and install spack for the fddaq release | |
env: | |
NIGHTLY_TAG: ${{needs.make_nightly_tag.outputs.tag}} | |
run: | | |
export BASE_RELEASE_DIR=/cvmfs/dunedaq-development.opensciencegrid.org/nightly/NB${NIGHTLY_TAG} | |
export DET_RELEASE_DIR=/cvmfs/dunedaq-development.opensciencegrid.org/nightly/NFD${NIGHTLY_TAG} | |
export OS=almalinux9 | |
source daq-release/.github/workflows/wf-setup-tools.sh | |
daq-release/scripts/spack/build-release.sh $BASE_RELEASE_DIR $DET_RELEASE_DIR fd $OS ${{ github.event.inputs.feature-branch }} | |
cd $DET_RELEASE_DIR/../ | |
tar_and_stage_release ${DET_RELEASE_TAG} | |
- name: upload spack nightly tarball for far detector release | |
uses: actions/upload-artifact@v3 | |
with: | |
name: nightly_fddaq | |
path: ${{ github.workspace }}/tarballs_for_upload/NFD${{needs.make_nightly_tag.outputs.tag}}.tar.gz | |
update_image: | |
name: update_spack_image_nightly | |
runs-on: dune-github | |
strategy: | |
matrix: | |
include: | |
- input_image: "ghcr.io/dune-daq/alma9-slim-externals:v2.1pt" | |
output_image: "ghcr.io/dune-daq/nightly-release-alma9" | |
tag: "development_v5_podman_test" | |
needs: [build_the_develop_release_spack, make_nightly_tag] | |
#environment: dockerhub | |
permissions: | |
packages: write | |
contents: read | |
steps: | |
- name: clean podman-build | |
run: | | |
mkdir -p ${GITHUB_WORKSPACE}/podman-build | |
rm -rf ${GITHUB_WORKSPACE}/podman-build/* | |
- name: Download spack nightly release tarball artifact for base release | |
uses: actions/download-artifact@v3 | |
with: | |
name: nightly_coredaq | |
path: ${{ github.workspace }}/podman-build | |
- name: Download spack nightly release tarball artifact for far detector release | |
uses: actions/download-artifact@v3 | |
with: | |
name: nightly_fddaq | |
path: ${{ github.workspace }}/podman-build | |
- name: prepare cvmfs mirror spack-nightly | |
env: | |
NIGHTLY_TAG: ${{needs.make_nightly_tag.outputs.tag}} | |
run: | | |
cd ${{ github.workspace }}/podman-build | |
mkdir -p nightly | |
echo "Setting up build directory $(pwd)" | |
cd nightly | |
base_tag="NB${NIGHTLY_TAG}" | |
tar xf ../${base_tag}.tar.gz | |
rm -rf ../${base_tag}.tar.gz | |
fddaq_tag="NFD${NIGHTLY_TAG}" | |
tar xf ../${fddaq_tag}.tar.gz | |
rm -rf ../${fddaq_tag}.tar.gz | |
ln -s ${fddaq_tag} last_fddaq | |
cd .. | |
echo "FROM "${{ matrix.input_image }} > Containerfile | |
echo 'MAINTAINER John Freeman "jcfree@fnal.gov"' >> Containerfile | |
echo "ENV REFRESHED_AT ${NIGHTLY_TAG}" >> Containerfile | |
echo "COPY --from=ghcr.io/dune-daq/pypi-repo:latest /cvmfs/dunedaq.opensciencegrid.org/pypi-repo /cvmfs/dunedaq.opensciencegrid.org/pypi-repo" >> Containerfile | |
echo "ADD nightly /cvmfs/dunedaq-development.opensciencegrid.org/nightly" >> Containerfile | |
echo 'ENTRYPOINT ["/bin/bash"]' >> Containerfile | |
echo "Containerfile path: $(readlink -f Containerfile)" | |
- name: Log in to the Container registry | |
uses: redhat-actions/podman-login@v1 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ env.USERNAME }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Extract metadata (tags, labels) for Docker | |
id: meta | |
uses: docker/metadata-action@v5 | |
with: | |
images: | | |
ghcr.io/dune-daq/nightly-release-alma9 | |
tags: | | |
type=raw,value=development_v5_podman_test | |
- name: Build Image | |
id: build_image | |
uses: redhat-actions/buildah-build@v2 | |
with: | |
image: ${{ matrix.output_image }} | |
tags: ${{ steps.meta.outputs.tags }} | |
labels: | | |
${{ steps.meta.outputs.labels }} | |
layers: true | |
context: podman-build | |
containerfiles: podman-build/Containerfile | |
oci: true | |
- name: Push to GitHub Container Repository | |
id: push_to_ghcr | |
uses: redhat-actions/push-to-registry@v2 | |
with: | |
image: ${{ steps.build_image.outputs.image }} | |
tags: ${{ steps.build_image.outputs.tags }} | |
registry: ${{ env.REGISTRY }}/${{ env.USERNAME}} | |
generate_dbt_setup_release_env: | |
name: generate_dbt_setup_release_env | |
runs-on: dune-github | |
needs: update_image | |
container: | |
image: ghcr.io/dune-daq/nightly-release-alma9:development_v5_podman_test | |
defaults: | |
run: | |
shell: bash | |
steps: | |
- name: create dbt-setup-release-env.sh and daq_app_rte.sh for fddaq | |
run: | | |
export DET=fd | |
source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh | |
setup_dbt latest || true | |
dbt-setup-release -n last_${DET}daq | |
declare -x > ${GITHUB_WORKSPACE}/${DET}daq-dbt-setup-release-env.sh | |
declare -f >> ${GITHUB_WORKSPACE}/${DET}daq-dbt-setup-release-env.sh | |
egrep "declare -x (PATH|.*_SHARE|CET_PLUGIN_PATH|DUNEDAQ_SHARE_PATH|LD_LIBRARY_PATH|LIBRARY_PATH|PYTHONPATH)=" ${GITHUB_WORKSPACE}/${DET}daq-dbt-setup-release-env.sh > ${GITHUB_WORKSPACE}/${DET}daq_app_rte.sh | |
- name: upload fddaq-dbt-setup-release-env.sh | |
uses: actions/upload-artifact@v3 | |
with: | |
name: fddaq-dbt_setup_release_env | |
path: ${{ github.workspace }}/fddaq-dbt-setup-release-env.sh | |
- name: upload fddaq_app_rte.sh | |
uses: actions/upload-artifact@v3 | |
with: | |
name: fddaq_app_rte | |
path: ${{ github.workspace }}/fddaq_app_rte.sh | |
publish_to_cvmfs: | |
name: publish to cvmfs | |
needs: generate_dbt_setup_release_env | |
runs-on: dune-github | |
steps: | |
- name: Log in to the Container registry | |
uses: redhat-actions/podman-login@v1 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ env.USERNAME }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: check_cvmfs_deployment | |
id: do_deployment | |
run: | | |
cvmfs_deployment_input=${{ github.event.inputs.cvmfs-deployment }} | |
cvmfs_deployment_flag=${cvmfs_deployment_input:-"yes"} | |
echo "Do cvmfs deployment: ${cvmfs_deployment_flag}" | |
[[ ${cvmfs_deployment_flag} == 'yes' ]] && exit 0 || exit 1 | |
continue-on-error: true | |
- name: download fddaq-dbt-setup-release-env.sh | |
if: steps.do_deployment.outcome == 'success' | |
uses: actions/download-artifact@v3 | |
with: | |
name: fddaq-dbt_setup_release_env | |
path: ${{ github.workspace }}/fddaq-release | |
- name: download fddaq_app_rte.sh | |
if: steps.do_deployment.outcome == 'success' | |
uses: actions/download-artifact@v3 | |
with: | |
name: fddaq_app_rte | |
path: ${{ github.workspace }}/fddaq-rte | |
- name: start podman container, and then rsync... | |
if: steps.do_deployment.outcome == 'success' | |
run: | | |
[[ ${DO_DEPLOYMENT} == 'no' ]] && exit 0 | |
cd $GITHUB_WORKSPACE | |
IMAGE="ghcr.io/dune-daq/nightly-release-alma9:development_v5_podman_test" | |
unique_name=$( date | tr " :" _ ) | |
podman pull $IMAGE | |
files_to_copy=$( podman run --rm --entrypoint ls $IMAGE /cvmfs/dunedaq-development.opensciencegrid.org/nightly ) | |
podman run --name $unique_name $IMAGE | |
mkdir $unique_name | |
for file in $files_to_copy; do | |
podman cp $unique_name:/cvmfs/dunedaq-development.opensciencegrid.org/nightly/$file $unique_name | |
done | |
podman rm $unique_name | |
dir_for_env_scripts=$( find $unique_name -mindepth 1 -maxdepth 1 -type d -name "NFD*" ) | |
cp ./fddaq-release/fddaq-dbt-setup-release-env.sh $dir_for_env_scripts/dbt-setup-release-env.sh | |
cp ./fddaq-rte/fddaq_app_rte.sh $dir_for_env_scripts/daq_app_rte.sh | |
#kinit -k -t $HOME/daq-nightly.keytab nightly-build/dune/daq.dunescience.org@FNAL.GOV | |
ssh -o StrictHostKeyChecking=no -l cvmfsdunedaqdev oasiscfs01.fnal.gov "cvmfs_server transaction dunedaq-development.opensciencegrid.org" | |
rsync -e "ssh -o StrictHostKeyChecking=no" -rlpt --stats $unique_name/* cvmfsdunedaqdev@oasiscfs01.fnal.gov:/cvmfs/dunedaq-development.opensciencegrid.org/nightly | |
ssh -o StrictHostKeyChecking=no -l cvmfsdunedaqdev oasiscfs01.fnal.gov "cvmfs_server publish dunedaq-development.opensciencegrid.org" | |
rm -rf $unique_name |