From ab1a17a19ae6fdfb97a4e22419325499548a5707 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Tue, 4 Apr 2023 14:50:12 +0200 Subject: [PATCH 1/7] Add script for generation of ARF report --- generate_arf.sh | 92 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100755 generate_arf.sh diff --git a/generate_arf.sh b/generate_arf.sh new file mode 100755 index 00000000..83051609 --- /dev/null +++ b/generate_arf.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# This script generate ARF results. +# Requirements: +# - cmake +# - make +# - openscap-utils +# - openscap-scanner +# - python3-pyyaml +# - python3-jinja2 +# - python3-setuptools +# - git +# - scap-security-guide +# Usage: ./generate_arf MODE FETCH PRODUCT ARF_FILE SKIP_BUILD +# MODE [latest, ssg] use scap-security-guide or latest content from github +# FETCH [yes, no] scanner fetch remote resources +# PRODUCT build or use security content for one specific product +# ARF_FILE Writes results to a given ARF_FILE. +# SKIP_BUILD [yes] Skip build of latest content(Have affect with mode latest). + + +set -e -o pipefail + + +build_content() { + product=$1 + + echo "Build - Start" + + git clone https://github.com/ComplianceAsCode/content.git + cd content/ + git checkout master + + ./build_product "${product}" + cd .. + echo "Build - Done" +} + +run_oscap_scan() { + ds=$1 + fetch=$2 + file=$3 + echo "Scans - Start" + oscap xccdf eval ${fetch} --profile "(all)" --results-arf ${file} ${ds} || EXIT_CODE=$? + echo $EXIT_CODE + if [ ! -f "$file" ]; then + echo "$file does not exist." + exit 2 + fi +} + + +if [ "$1" = "" ]; then + echo "ERROR: Missing MODE parameter!" + exit 1 +fi + + +if [ "$2" = "" ]; then + echo "ERROR: Missing FETCH parameter!" + exit 1 +fi + + +if [ "$3" = "" ]; then + echo "ERROR: Missing PRODUCT parameter!" + exit 1 +fi + +if [ "$4" = "" ]; then + echo "ERROR: Missing PRODUCT parameter!" + exit 1 +fi + +file=$4 +product=$3 + +fetch="--fetch-remote-resources" +if [ "$2" = "no" ]; then + fetch="" +fi + + +if [ "$1" = "latest" ]; then + if [ "$5" != "yes" ]; then + build_content "${product}" + fi + run_oscap_scan "./content/build/ssg-${product}-ds.xml" "${fetch}" "${file}" +fi + +if [ "$1" = "ssg" ]; then + run_oscap_scan "/usr/share/xml/scap/ssg/content/ssg-${product}-ds.xml" "${fetch}" "${file}" +fi From d25a3201284398b10cbd10ac882acc0bcc04657d Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Tue, 4 Apr 2023 14:52:49 +0200 Subject: [PATCH 2/7] Modify the smoke test to use different ARF reports --- tests/smoke.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/smoke.sh b/tests/smoke.sh index e6234463..cee51388 100755 --- a/tests/smoke.sh +++ b/tests/smoke.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash # Test of the basic function +# Usage: ./smoke.sh path_to_arf +# path_to_arf (Default: ./test_data/arf-report.xml) Path where is tested ARF file set -e -o pipefail +path_to_arf=$1 +if [ "$path_to_arf" = "" ]; then + path_to_arf="./test_data/arf-report.xml" +fi + # Generate report -oscap-report < ./test_data/arf-report.xml > report.html +oscap-report < "${path_to_arf}" > report.html # Search for some rule ID in the report grep -q "xccdf_org\.ssgproject\.content_rule_enable_fips_mode" report.html From adae6a02df0ec362010e68060d6596344785fd8a Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Tue, 4 Apr 2023 20:18:50 +0200 Subject: [PATCH 3/7] Add tmt plan --- plans/integration.fmf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 plans/integration.fmf diff --git a/plans/integration.fmf b/plans/integration.fmf new file mode 100644 index 00000000..c80dae90 --- /dev/null +++ b/plans/integration.fmf @@ -0,0 +1,25 @@ +summary: Test integration with latest versions of content +discover+: + filter: tag:integration +prepare: + - name: Install packages require for generation ARF files + how: install + package: + - cmake + - make + - openscap-utils + - openscap-scanner + - python3-pyyaml + - python3-jinja2 + - python3-setuptools + - git + - scap-security-guide + - name: Generate ARF files + how: shell + script: + - ./generate_arf.sh ssg no fedora ${TMT_PLAN_DATA}/arf.xml + - ./generate_arf.sh ssg yes fedora ${TMT_PLAN_DATA}/arf_fetch-remote-resources.xml + - ./generate_arf.sh latest no fedora ${TMT_PLAN_DATA}/arf-latest.xml + - ./generate_arf.sh latest yes fedora ${TMT_PLAN_DATA}/arf_fetch-remote-resources-latest.xml yes +execute: + how: tmt From 2036b60ee4ebfbf523668bec6de4801e1b3d48e1 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Tue, 4 Apr 2023 20:19:07 +0200 Subject: [PATCH 4/7] Add tmt tests --- tests/integration.fmf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/integration.fmf diff --git a/tests/integration.fmf b/tests/integration.fmf new file mode 100644 index 00000000..fbf584d8 --- /dev/null +++ b/tests/integration.fmf @@ -0,0 +1,31 @@ +require: + - openscap-report + +description: | + Test integration with latest and released version of content. This tests require files generated in prepare stage named "Generate ARF files". + Test "arf" uses ARF file tahat were generated from datastream provided by scap-security-guide package. + Test "arf-fetch-remote-resources" uses ARF file tahat were generated from datastream provided by scap-security-guide package and scan were performed with parameter "--fetch-remote-resources". + Test "latest-arf" uses ARF file tahat were generated from datastream provided by Github repository ComplianceAsCode/content. + Test "latest-arf-fetch-remote-resources" uses ARF file tahat were generated from datastream provided by Github repository ComplianceAsCode/content and scan were performed with parameter "--fetch-remote-resources". + + +/arf: + summary: scap-security-guide + test: ./smoke.sh ${TMT_PLAN_DATA}/arf.xml + +/arf-fetch-remote-resources: + summary: scap-security-guide, --fetch-remote-resources + test: ./smoke.sh ${TMT_PLAN_DATA}/arf_fetch-remote-resources.xml + +/latest-arf: + summary: ComplianceAsCode/content + test: ./smoke.sh ${TMT_PLAN_DATA}/arf-latest.xml + +/latest-arf-fetch-remote-resources: + summary: ComplianceAsCode/content, --fetch-remote-resources + test: ./smoke.sh ${TMT_PLAN_DATA}/arf_fetch-remote-resources-latest.xml + + +tag: + - integration +tier: 1 From 3a994ae9df46c4fd73fc1af4d8ac2d1862d05e4e Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Wed, 5 Apr 2023 15:48:44 +0200 Subject: [PATCH 5/7] Add weekly execution of integration test --- .github/workflows/weekly-integration-test.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/weekly-integration-test.yml diff --git a/.github/workflows/weekly-integration-test.yml b/.github/workflows/weekly-integration-test.yml new file mode 100644 index 00000000..764a2049 --- /dev/null +++ b/.github/workflows/weekly-integration-test.yml @@ -0,0 +1,20 @@ +name: "Weekly Integration Test" + +on: + push: + branches: ["main"] + schedule: + - cron: '0 21 * * 6' +jobs: + integration-test: + name: Integration test + runs-on: ubuntu-latest + container: + image: fedora:latest + steps: + - name: Install Deps + run: dnf install -y tmt + - name: Checkout + uses: actions/checkout@v3 + - name: Execute test + run: tmt run --all provision --how=local From 837931cdac1de8b1d47ee2aa13055bc65a96dbcb Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Wed, 5 Apr 2023 18:43:05 +0200 Subject: [PATCH 6/7] Run only smoke test in Testing Farm --- .packit.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.packit.yaml b/.packit.yaml index 075824be..0d834d37 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -50,6 +50,7 @@ jobs: - job: tests trigger: pull_request identifier: tests-all + tmt_plan: smoke targets: - fedora-all - epel-9 From 279de0feb65ab099d5ca1a4a26b6664619e85bfb Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Thu, 13 Apr 2023 15:14:16 +0200 Subject: [PATCH 7/7] Determine which product use --- .github/workflows/weekly-integration-test.yml | 2 +- plans/integration.fmf | 30 ++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/weekly-integration-test.yml b/.github/workflows/weekly-integration-test.yml index 764a2049..7de9e40c 100644 --- a/.github/workflows/weekly-integration-test.yml +++ b/.github/workflows/weekly-integration-test.yml @@ -17,4 +17,4 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Execute test - run: tmt run --all provision --how=local + run: tmt -c distro=fedora run --all provision --how=local diff --git a/plans/integration.fmf b/plans/integration.fmf index c80dae90..9d9bb7e8 100644 --- a/plans/integration.fmf +++ b/plans/integration.fmf @@ -1,6 +1,27 @@ summary: Test integration with latest versions of content discover+: filter: tag:integration +adjust: + - when: distro == fedora + environment: + PRODUCT: fedora + TO_BUILD_PRODUCT: fedora + - when: distro == rhel-9 + environment: + PRODUCT: rhel9 + TO_BUILD_PRODUCT: rhel9 + - when: distro == rhel-8 + environment: + PRODUCT: rhel8 + TO_BUILD_PRODUCT: rhel8 + - when: distro == centos-8 + environment: + PRODUCT: centos8 + TO_BUILD_PRODUCT: rhel8 + - when: distro == centos-9 or distro == centos-stream-9 + environment: + PRODUCT: cs9 + TO_BUILD_PRODUCT: rhel9 prepare: - name: Install packages require for generation ARF files how: install @@ -17,9 +38,10 @@ prepare: - name: Generate ARF files how: shell script: - - ./generate_arf.sh ssg no fedora ${TMT_PLAN_DATA}/arf.xml - - ./generate_arf.sh ssg yes fedora ${TMT_PLAN_DATA}/arf_fetch-remote-resources.xml - - ./generate_arf.sh latest no fedora ${TMT_PLAN_DATA}/arf-latest.xml - - ./generate_arf.sh latest yes fedora ${TMT_PLAN_DATA}/arf_fetch-remote-resources-latest.xml yes + - ./generate_arf.sh ssg no ${PRODUCT} ${TMT_PLAN_DATA}/arf.xml + - ./generate_arf.sh ssg yes ${PRODUCT} ${TMT_PLAN_DATA}/arf_fetch-remote-resources.xml + - ./generate_arf.sh latest no ${TO_BUILD_PRODUCT} ${TMT_PLAN_DATA}/arf-latest.xml + - ./generate_arf.sh latest yes ${TO_BUILD_PRODUCT} ${TMT_PLAN_DATA}/arf_fetch-remote-resources-latest.xml yes + execute: how: tmt