CFE-4623: Fix modules_presence to handle findfiles() trailing slash behavior (3.27)

.github/workflows/bootstrap_policy_run_check.yml

@@ -3,11 +3,29 @@ on:
 
 jobs:
   bootstrap_policy_run_check:
+    strategy:
+      fail-fast: false
+      matrix:
+        cfengine_version: [ "3.21.8", "3.24.3", "3.27.0", "master" ]
+
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           path: masterfiles
+      - name: Get Togethers
+        uses: cfengine/together-javascript-action@main
+        id: together
+        with:
+          myToken: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout Core
+        if: ${{ steps.together.outputs.core != null }}
+        uses: actions/checkout@v4
+        with:
+          repository: cfengine/core
+          path: core
+          ref: ${{steps.together.outputs.core || github.base_ref || github.ref}}
+          submodules: recursive
       - name: Install, bootstrap, policy run
-        run: masterfiles/ci/docker-bootstrap-policy-run.sh
+        run: masterfiles/ci/docker-bootstrap-policy-run.sh ${{ matrix.cfengine_version }}

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ on:
   # run this workflow on pull_request activity
   # this includes opening and pushing more commits
   pull_request:
-    branches: [ master, 3.24.x, 3.21.x ]
+    branches: [ master, 3.27.x, 3.24.x ]
 
 jobs:
   style_check:

cfe_internal/update/update_policy.cf

@@ -801,22 +801,26 @@ bundle agent modules_presence
     # TODO: Remove this once sys.moduledir is always available (3.26+ only)
     "moduledir" string => ifelse(isvariable("sys.moduledir"), "$(sys.moduledir)", "$(sys.workdir)/modules");
     "_vendored_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored$(const.dirsep)";
+    "_vendored_dir_filter" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored";
     "_override_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)";
     "_custom_template_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)mustache$(const.dirsep)";
     "_vendored_paths" slist => findfiles("$(_vendored_dir)*.mustache");
     "_custom_template_paths" slist => findfiles("$(_custom_template_dir)*.mustache"), if => isdir( "$(_custom_template_dir)" );
-    "_package_paths" slist => filter("$(_override_dir)vendored", _package_paths_tmp, "false", "true", 999);
+    "_package_paths_tmp" slist => findfiles("${_override_dir}*"),
+      comment => "We get a temp list of files that we have to filter out the vendored sub directory.";
 
     windows::
-      "_package_paths_tmp" slist => findfiles("$(_override_dir)*");
+      "_package_paths" slist => filter("\Q${_vendored_dir_filter}\E.*", _package_paths_tmp, "true", "true", 999),
+        comment => "The list will include the vendored sub-directory that we don't want so remove it.";
       "_vendored_modules" slist => maplist(regex_replace("$(this)", "\Q$(_vendored_dir)\E(.*).mustache", "$1", "g"), @(_vendored_paths));
       "_override_modules" slist => maplist(regex_replace("$(this)", "\Q$(_override_dir)\E(.*)", "$1", "g"), @(_package_paths));
       # replace single backslashes in a windows path with double-backslashes
       # to avoid problems with things like `C:\Program Files` and `\promises`
       # causing PCRE to try and interpret special escape sequences.
       "_not_vendored_modules_pathname_regex" string => regex_replace("$(sys.inputdir)$(const.dirsep)modules$(const.dirsep)(?!packages$(const.dirsep)vendored).*","\\\\","\\\\\\\\","g");
     !windows::
-      "_package_paths_tmp" slist => findfiles("$(_override_dir)*");
+      "_package_paths" slist => filter("${_vendored_dir_filter}.*", _package_paths_tmp, "true", "true", 999),
+        comment => "The list will include the vendored sub-directory that we don't want so remove it.";
       "_vendored_modules" slist => maplist(regex_replace("$(this)", "$(_vendored_dir)(.*).mustache", "$1", "g"), @(_vendored_paths));
       "_override_modules" slist => maplist(regex_replace("$(this)", "$(_override_dir)(.*)", "$1", "g"), @(_package_paths));
       "_custom_template_modules" slist => maplist(regex_replace("$(this)", "$(_custom_template_dir)(.*).mustache", "$1", "g"), @(_custom_template_paths));
@@ -867,14 +871,13 @@ bundle agent modules_presence
   reports:
     DEBUG::
       "_override_dir: $(_override_dir)";
-      "_package_paths_tmp: $(with)" with => storejson(_package_paths_tmp);
+      "_package_paths: $(with)" with => storejson(_package_paths);
       "_not_vendored_modules_pathname_regex: $(_not_vendored_modules_pathname_regex)";
       "_vendored_modules: $(_vendored_modules)";
       "_override_modules: $(_override_modules)";
       "_vendored_dir: $(_vendored_dir)";
       "_vendored_paths: $(_vendored_paths)";
       "_override_dir: $(_override_dir)";
-      "_package_paths: $(_package_paths)";
       "override_vendored_module_$(_vendored_modules)"
         if => "override_vendored_module_$(_vendored_modules)";
       "override_module_$(_override_modules)"

ci/bootstrap-policy-run.cfremote.Dockerfile

@@ -0,0 +1,5 @@
+ARG CFENGINE_VERSION="master"
+FROM debian
+RUN apt update && apt upgrade -y
+RUN apt install -y pipx sudo make automake autoconf git procps python3
+RUN pipx install cf-remote

ci/bootstrap-policy-run.sh

@@ -1,8 +1,16 @@
 #!/usr/bin/env sh
 set -ex
-./autogen.sh --prefix=/var/lib/cfengine
+if [ -f ../core/ci/install.sh ]; then
+  ../core/ci/install.sh
+else
+  # here we use community so that masterfiles has less errors when bootstrapping as it expects an enterprise hub with the -nova package
+  PATH=/root/.local/bin:$PATH cf-remote --version "$CFENGINE_VERSION" install --edition community --clients localhost
+fi
+./autogen.sh
 make install
+export PATH=/var/cfengine/bin:$PATH
 which cf-agent
+ps -efl | grep cf- # debug cf-serverd already running somehow?
 cf-agent -IB $(hostname -i) | tee bootstrap.log
 cf-agent -KIf update.cf | tee update.log
 cf-agent -KI | tee promise.log

ci/bootstrap-policy-run.source.Dockerfile

@@ -0,0 +1,6 @@
+ARG CFENGINE_VERSION="master"
+FROM debian
+COPY core /core
+RUN apt update && apt upgrade -y
+# need python3 for apt_get package module to avoid errors
+RUN apt install -y sudo make automake autoconf git python3 procps

ci/docker-bootstrap-policy-run.sh

@@ -5,27 +5,41 @@ set -ex
 COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")"
 # NTECH_ROOT should be the same, but if available use it so user can do their own thing.
 NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+CFENGINE_VERSION=${1:-master}
+export CFENGINE_VERSION
 
 cd "${NTECH_ROOT}/masterfiles"
 
 # cleanup
 rm -f update.log bootstrap.log promise.log
-if docker ps | grep mpf; then
-  docker stop mpf
+image_name=bootstrap-${CFENGINE_VERSION}
+if docker ps | grep "$image_name"; then
+  docker stop "$image_name"
 fi
-if docker ps -a | grep mpf; then
-  docker ps -a | grep mpf | awk '{print $1}' | xargs docker rm
+if docker ps -a | grep "$image_name"; then
+  docker ps -a | grep "$image_name" | awk '{print $1}' | xargs docker rm
 fi
-if docker images | grep mpf; then
-  docker rmi mpf
+if docker images | grep "$image_name"; then
+  docker rmi "$image_name"
+fi
+
+if [ -d "${NTECH_ROOT}"/core ]; then
+  docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.source.Dockerfile  "${NTECH_ROOT}"
+else
+  docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.cfremote.Dockerfile  "${NTECH_ROOT}"
 fi
 
 # run the test
-docker build -t mpf -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.Dockerfile  "${NTECH_ROOT}"/masterfiles
-docker run --workdir /mpf --volume "${NTECH_ROOT}"/masterfiles:/mpf --tty mpf sh /mpf/ci/bootstrap-policy-run.sh
-if grep error *.log; then
+docker run -e CFENGINE_VERSION --workdir /masterfiles --volume "${NTECH_ROOT}"/masterfiles:/masterfiles --tty "$image_name" sh /masterfiles/ci/bootstrap-policy-run.sh
+
+if grep error ./*.log; then
   echo "fail"
   exit 1
 else
   echo "success"
 fi
+
+if [ ! -f bootstrap.log ] || [ ! -f promise.log ] || [ ! -f update.log ]; then
+  echo "No log files. Fail."
+  exit 23
+fi