diff --git a/.github/workflows/bootstrap_policy_run_check.yml b/.github/workflows/bootstrap_policy_run_check.yml index c356cfd3ff..ec9ea7bc7b 100644 --- a/.github/workflows/bootstrap_policy_run_check.yml +++ b/.github/workflows/bootstrap_policy_run_check.yml @@ -3,11 +3,29 @@ on: jobs: bootstrap_policy_run_check: + strategy: + fail-fast: false + matrix: + cfengine_version: [ "3.21.8", "3.24.3", "3.27.0", "master" ] + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 with: path: masterfiles + - name: Get Togethers + uses: cfengine/together-javascript-action@main + id: together + with: + myToken: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout Core + if: ${{ steps.together.outputs.core != null }} + uses: actions/checkout@v4 + with: + repository: cfengine/core + path: core + ref: ${{steps.together.outputs.core || github.base_ref || github.ref}} + submodules: recursive - name: Install, bootstrap, policy run - run: masterfiles/ci/docker-bootstrap-policy-run.sh + run: masterfiles/ci/docker-bootstrap-policy-run.sh ${{ matrix.cfengine_version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b143b97367..367b031ce5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: # run this workflow on pull_request activity # this includes opening and pushing more commits pull_request: - branches: [ master, 3.24.x, 3.21.x ] + branches: [ master, 3.27.x, 3.24.x ] jobs: style_check: diff --git a/cfe_internal/update/update_policy.cf b/cfe_internal/update/update_policy.cf index d4f4302549..7f8754daf3 100644 --- a/cfe_internal/update/update_policy.cf +++ b/cfe_internal/update/update_policy.cf @@ -801,14 +801,17 @@ bundle agent modules_presence # TODO: Remove this once sys.moduledir is always available (3.26+ only) "moduledir" string => ifelse(isvariable("sys.moduledir"), "$(sys.moduledir)", "$(sys.workdir)/modules"); "_vendored_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored$(const.dirsep)"; + "_vendored_dir_filter" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored"; "_override_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)"; "_custom_template_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)mustache$(const.dirsep)"; "_vendored_paths" slist => findfiles("$(_vendored_dir)*.mustache"); "_custom_template_paths" slist => findfiles("$(_custom_template_dir)*.mustache"), if => isdir( "$(_custom_template_dir)" ); - "_package_paths" slist => filter("$(_override_dir)vendored", _package_paths_tmp, "false", "true", 999); + "_package_paths_tmp" slist => findfiles("${_override_dir}*"), + comment => "We get a temp list of files that we have to filter out the vendored sub directory."; windows:: - "_package_paths_tmp" slist => findfiles("$(_override_dir)*"); + "_package_paths" slist => filter("\Q${_vendored_dir_filter}\E.*", _package_paths_tmp, "true", "true", 999), + comment => "The list will include the vendored sub-directory that we don't want so remove it."; "_vendored_modules" slist => maplist(regex_replace("$(this)", "\Q$(_vendored_dir)\E(.*).mustache", "$1", "g"), @(_vendored_paths)); "_override_modules" slist => maplist(regex_replace("$(this)", "\Q$(_override_dir)\E(.*)", "$1", "g"), @(_package_paths)); # replace single backslashes in a windows path with double-backslashes @@ -816,7 +819,8 @@ bundle agent modules_presence # causing PCRE to try and interpret special escape sequences. "_not_vendored_modules_pathname_regex" string => regex_replace("$(sys.inputdir)$(const.dirsep)modules$(const.dirsep)(?!packages$(const.dirsep)vendored).*","\\\\","\\\\\\\\","g"); !windows:: - "_package_paths_tmp" slist => findfiles("$(_override_dir)*"); + "_package_paths" slist => filter("${_vendored_dir_filter}.*", _package_paths_tmp, "true", "true", 999), + comment => "The list will include the vendored sub-directory that we don't want so remove it."; "_vendored_modules" slist => maplist(regex_replace("$(this)", "$(_vendored_dir)(.*).mustache", "$1", "g"), @(_vendored_paths)); "_override_modules" slist => maplist(regex_replace("$(this)", "$(_override_dir)(.*)", "$1", "g"), @(_package_paths)); "_custom_template_modules" slist => maplist(regex_replace("$(this)", "$(_custom_template_dir)(.*).mustache", "$1", "g"), @(_custom_template_paths)); @@ -867,14 +871,13 @@ bundle agent modules_presence reports: DEBUG:: "_override_dir: $(_override_dir)"; - "_package_paths_tmp: $(with)" with => storejson(_package_paths_tmp); + "_package_paths: $(with)" with => storejson(_package_paths); "_not_vendored_modules_pathname_regex: $(_not_vendored_modules_pathname_regex)"; "_vendored_modules: $(_vendored_modules)"; "_override_modules: $(_override_modules)"; "_vendored_dir: $(_vendored_dir)"; "_vendored_paths: $(_vendored_paths)"; "_override_dir: $(_override_dir)"; - "_package_paths: $(_package_paths)"; "override_vendored_module_$(_vendored_modules)" if => "override_vendored_module_$(_vendored_modules)"; "override_module_$(_override_modules)" diff --git a/ci/bootstrap-policy-run.Dockerfile b/ci/bootstrap-policy-run.Dockerfile deleted file mode 100644 index fc91a79f52..0000000000 --- a/ci/bootstrap-policy-run.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM alpine -RUN apk update && apk add cfengine make automake autoconf git diff --git a/ci/bootstrap-policy-run.cfremote.Dockerfile b/ci/bootstrap-policy-run.cfremote.Dockerfile new file mode 100644 index 0000000000..500318a04e --- /dev/null +++ b/ci/bootstrap-policy-run.cfremote.Dockerfile @@ -0,0 +1,5 @@ +ARG CFENGINE_VERSION="master" +FROM debian +RUN apt update && apt upgrade -y +RUN apt install -y pipx sudo make automake autoconf git procps python3 +RUN pipx install cf-remote diff --git a/ci/bootstrap-policy-run.sh b/ci/bootstrap-policy-run.sh index b2143e3242..ab10f38b1c 100755 --- a/ci/bootstrap-policy-run.sh +++ b/ci/bootstrap-policy-run.sh @@ -1,8 +1,16 @@ #!/usr/bin/env sh set -ex -./autogen.sh --prefix=/var/lib/cfengine +if [ -f ../core/ci/install.sh ]; then + ../core/ci/install.sh +else + # here we use community so that masterfiles has less errors when bootstrapping as it expects an enterprise hub with the -nova package + PATH=/root/.local/bin:$PATH cf-remote --version "$CFENGINE_VERSION" install --edition community --clients localhost +fi +./autogen.sh make install +export PATH=/var/cfengine/bin:$PATH which cf-agent +ps -efl | grep cf- # debug cf-serverd already running somehow? cf-agent -IB $(hostname -i) | tee bootstrap.log cf-agent -KIf update.cf | tee update.log cf-agent -KI | tee promise.log diff --git a/ci/bootstrap-policy-run.source.Dockerfile b/ci/bootstrap-policy-run.source.Dockerfile new file mode 100644 index 0000000000..1e51578ac2 --- /dev/null +++ b/ci/bootstrap-policy-run.source.Dockerfile @@ -0,0 +1,6 @@ +ARG CFENGINE_VERSION="master" +FROM debian +COPY core /core +RUN apt update && apt upgrade -y +# need python3 for apt_get package module to avoid errors +RUN apt install -y sudo make automake autoconf git python3 procps diff --git a/ci/docker-bootstrap-policy-run.sh b/ci/docker-bootstrap-policy-run.sh index 6db6215712..1ed8b5cd57 100755 --- a/ci/docker-bootstrap-policy-run.sh +++ b/ci/docker-bootstrap-policy-run.sh @@ -5,27 +5,41 @@ set -ex COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")" # NTECH_ROOT should be the same, but if available use it so user can do their own thing. NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} +CFENGINE_VERSION=${1:-master} +export CFENGINE_VERSION cd "${NTECH_ROOT}/masterfiles" # cleanup rm -f update.log bootstrap.log promise.log -if docker ps | grep mpf; then - docker stop mpf +image_name=bootstrap-${CFENGINE_VERSION} +if docker ps | grep "$image_name"; then + docker stop "$image_name" fi -if docker ps -a | grep mpf; then - docker ps -a | grep mpf | awk '{print $1}' | xargs docker rm +if docker ps -a | grep "$image_name"; then + docker ps -a | grep "$image_name" | awk '{print $1}' | xargs docker rm fi -if docker images | grep mpf; then - docker rmi mpf +if docker images | grep "$image_name"; then + docker rmi "$image_name" +fi + +if [ -d "${NTECH_ROOT}"/core ]; then + docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.source.Dockerfile "${NTECH_ROOT}" +else + docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.cfremote.Dockerfile "${NTECH_ROOT}" fi # run the test -docker build -t mpf -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.Dockerfile "${NTECH_ROOT}"/masterfiles -docker run --workdir /mpf --volume "${NTECH_ROOT}"/masterfiles:/mpf --tty mpf sh /mpf/ci/bootstrap-policy-run.sh -if grep error *.log; then +docker run -e CFENGINE_VERSION --workdir /masterfiles --volume "${NTECH_ROOT}"/masterfiles:/masterfiles --tty "$image_name" sh /masterfiles/ci/bootstrap-policy-run.sh + +if grep error ./*.log; then echo "fail" exit 1 else echo "success" fi + +if [ ! -f bootstrap.log ] || [ ! -f promise.log ] || [ ! -f update.log ]; then + echo "No log files. Fail." + exit 23 +fi