[WIP] add possibility for generating zstd-compressed tarballs

bot/build.sh

@@ -272,9 +272,9 @@ source $software_layer_dir/init/eessi_defaults
 # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
 # appended
 if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
-    export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+    export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.tar.zst" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
 else
-    export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+    export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.zst" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
 fi
 
 # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
@@ -288,8 +288,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
 echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
-echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}"
 $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
-                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

bot/check-build.sh

@@ -17,7 +17,7 @@
 
 #  - SUCCESS (all of)
 #    - working directory contains slurm-JOBID.out file
-#    - working directory contains eessi*tar.gz
+#    - working directory contains eessi*tar*
 #    - no message FATAL
 #    - no message ERROR
 #    - no message FAILED
@@ -165,19 +165,19 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
 fi
 
 if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
-    TGZ=-1
+    TARBALL_CREATED=-1
     TARBALL=
     if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-      GP_tgz_created="\.tar\.gz created!"
-      grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
+      GP_tarball_created="\.tar.* created!"
+      grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tarball_created}" | sort -u)
       if [[ $? -eq 0 ]]; then
-          TGZ=1
+          TARBALL_CREATED=1
           TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
       else
-          TGZ=0
+          TARBALL_CREATED=0
       fi
       # have to be careful to not add searched for pattern into slurm out file
-      [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+      [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tarball_created}"'"
       [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
     fi
 fi
@@ -190,7 +190,7 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
-    [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+    [[ ${VERBOSE} -ne 0 ]] && echo "  TARBALL_CREATED: $([[ $TARBALL -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 fi
 
 # Here, we try to do some additional analysis on the output file
@@ -219,7 +219,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${FAILED} -eq 0 ]] && \
    [[ ${MISSING} -eq 0 ]] && \
    [[ ${NO_MISSING} -eq 1 ]] && \
-   [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \
+   [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TARBALL_CREATED} -eq 1 ]] && \
    [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then
     # SUCCESS
     status="SUCCESS"
@@ -429,9 +429,9 @@ failure_msg="no message matching <code>${GP_no_missing}</code>"
 comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 
 if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
-    success_msg="found message matching <code>${GP_tgz_created}</code>"
-    failure_msg="no message matching <code>${GP_tgz_created}</code>"
-    comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+    success_msg="found message matching <code>${GP_tarball_created}</code>"
+    failure_msg="no message matching <code>${GP_tarball_created}</code>"
+    comment_details_list=${comment_details_list}$(add_detail ${TARBALL_CREATED} 1 "${success_msg}" "${failure_msg}")
 fi
 
 # Now, do the actual replacement of __DETAILS_FMT__

create_tarball.sh

@@ -12,7 +12,7 @@ eessi_tmpdir=$1
 eessi_version=$2
 cpu_arch_subdir=$3
 accel_subdir=$4
-target_tgz=$5
+target_tarball=$5
 
 tmpdir=`mktemp -d`
 echo ">> tmpdir: $tmpdir"
@@ -114,10 +114,9 @@ fi
 
 topdir=${cvmfs_repo}/versions/
 
-echo ">> Creating tarball ${target_tgz} from ${topdir}..."
-tar cfvz ${target_tgz} -C ${topdir} --files-from=${files_list}
-
-echo ${target_tgz} created!
+echo ">> Creating tarball ${target_tarball} from ${topdir}..."
+tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list}
+echo ${target_tarball} created!
 
 echo ">> Cleaning up tmpdir ${tmpdir}..."
 rm -r ${tmpdir}

easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml

@@ -0,0 +1,2 @@
+easyconfigs:
+  - cowsay-3.04.eb