Merge pull request #151 from Honny1/integration-tests

Integration tests

.github/workflows/weekly-integration-test.yml

@@ -0,0 +1,20 @@
+name: "Weekly Integration Test"
+
+on:
+  push:
+    branches: ["main"]
+  schedule:
+    - cron: '0 21 * * 6'
+jobs:
+  integration-test:
+    name: Integration test
+    runs-on: ubuntu-latest
+    container:
+      image: fedora:latest
+    steps:
+      - name: Install Deps
+        run: dnf install -y tmt
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Execute test
+        run: tmt -c distro=fedora run --all provision --how=local

.packit.yaml

@@ -48,6 +48,7 @@ jobs:
 - job: tests
   trigger: pull_request
   identifier: tests-all
+  tmt_plan: smoke
   targets:
     - fedora-all
     - epel-9

generate_arf.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# This script generate ARF results.
+# Requirements:
+#  - cmake
+#  - make
+#  - openscap-utils
+#  - openscap-scanner
+#  - python3-pyyaml
+#  - python3-jinja2
+#  - python3-setuptools
+#  - git
+#  - scap-security-guide
+# Usage: ./generate_arf MODE FETCH PRODUCT ARF_FILE SKIP_BUILD
+#   MODE        [latest, ssg] use scap-security-guide or latest content from github
+#   FETCH       [yes, no] scanner fetch remote resources 
+#   PRODUCT     build or use security content for one specific product
+#   ARF_FILE    Writes results to a given ARF_FILE.
+#   SKIP_BUILD  [yes] Skip build of latest content(Have affect with mode latest).
+
+
+set -e -o pipefail
+
+
+build_content() {
+    product=$1
+
+    echo "Build - Start"
+
+    git clone https://github.com/ComplianceAsCode/content.git
+    cd content/
+    git checkout master
+
+    ./build_product "${product}"
+    cd ..
+    echo "Build - Done"
+}
+
+run_oscap_scan() {
+    ds=$1
+    fetch=$2
+    file=$3
+    echo "Scans - Start"
+    oscap xccdf eval ${fetch} --profile "(all)" --results-arf ${file} ${ds} || EXIT_CODE=$?
+    echo $EXIT_CODE
+    if [ ! -f "$file" ]; then
+        echo "$file does not exist."
+        exit 2
+    fi
+}
+
+
+if [ "$1" = "" ]; then
+    echo "ERROR: Missing MODE parameter!"
+    exit 1
+fi
+
+
+if [ "$2" = "" ]; then
+    echo "ERROR: Missing FETCH parameter!"
+    exit 1
+fi
+
+
+if [ "$3" = "" ]; then
+    echo "ERROR: Missing PRODUCT parameter!"
+    exit 1
+fi
+
+if [ "$4" = "" ]; then
+    echo "ERROR: Missing PRODUCT parameter!"
+    exit 1
+fi
+
+file=$4
+product=$3
+
+fetch="--fetch-remote-resources"
+if [ "$2" = "no" ]; then
+    fetch=""
+fi
+
+
+if [ "$1" = "latest" ]; then
+    if [ "$5" != "yes" ]; then
+        build_content "${product}"
+    fi
+    run_oscap_scan "./content/build/ssg-${product}-ds.xml" "${fetch}" "${file}"
+fi
+
+if [ "$1" = "ssg" ]; then
+    run_oscap_scan "/usr/share/xml/scap/ssg/content/ssg-${product}-ds.xml" "${fetch}" "${file}"
+fi

plans/integration.fmf

@@ -0,0 +1,47 @@
+summary: Test integration with latest versions of content
+discover+:
+  filter: tag:integration
+adjust:
+  - when: distro == fedora
+    environment:
+      PRODUCT: fedora
+      TO_BUILD_PRODUCT: fedora
+  - when: distro == rhel-9
+    environment:
+      PRODUCT: rhel9
+      TO_BUILD_PRODUCT: rhel9
+  - when: distro == rhel-8
+    environment:
+      PRODUCT: rhel8
+      TO_BUILD_PRODUCT: rhel8
+  - when: distro == centos-8
+    environment:
+      PRODUCT: centos8
+      TO_BUILD_PRODUCT: rhel8
+  - when: distro == centos-9 or distro == centos-stream-9
+    environment:
+      PRODUCT: cs9
+      TO_BUILD_PRODUCT: rhel9
+prepare:
+  - name: Install packages require for generation ARF files
+    how: install
+    package:
+      - cmake
+      - make
+      - openscap-utils
+      - openscap-scanner
+      - python3-pyyaml
+      - python3-jinja2
+      - python3-setuptools
+      - git
+      - scap-security-guide
+  - name: Generate ARF files
+    how: shell
+    script: 
+      - ./generate_arf.sh ssg no ${PRODUCT} ${TMT_PLAN_DATA}/arf.xml
+      - ./generate_arf.sh ssg yes ${PRODUCT} ${TMT_PLAN_DATA}/arf_fetch-remote-resources.xml
+      - ./generate_arf.sh latest no ${TO_BUILD_PRODUCT} ${TMT_PLAN_DATA}/arf-latest.xml
+      - ./generate_arf.sh latest yes ${TO_BUILD_PRODUCT} ${TMT_PLAN_DATA}/arf_fetch-remote-resources-latest.xml yes
+
+execute:
+    how: tmt

tests/integration.fmf

@@ -0,0 +1,31 @@
+require:
+  - openscap-report
+
+description: |
+  Test integration with latest and released version of content. This tests require files generated in prepare stage named "Generate ARF files".
+  Test "arf" uses ARF file tahat were generated from datastream provided by scap-security-guide package.
+  Test "arf-fetch-remote-resources" uses ARF file tahat were generated from datastream provided by scap-security-guide package and scan were performed with parameter "--fetch-remote-resources".
+  Test "latest-arf" uses ARF file tahat were generated from datastream provided by Github repository ComplianceAsCode/content.
+  Test "latest-arf-fetch-remote-resources" uses ARF file tahat were generated from datastream provided by Github repository ComplianceAsCode/content and scan were performed with parameter "--fetch-remote-resources".
+
+
+/arf:
+    summary: scap-security-guide
+    test: ./smoke.sh ${TMT_PLAN_DATA}/arf.xml
+
+/arf-fetch-remote-resources:
+    summary: scap-security-guide, --fetch-remote-resources
+    test: ./smoke.sh ${TMT_PLAN_DATA}/arf_fetch-remote-resources.xml
+
+/latest-arf:
+    summary: ComplianceAsCode/content
+    test: ./smoke.sh ${TMT_PLAN_DATA}/arf-latest.xml
+
+/latest-arf-fetch-remote-resources:
+    summary: ComplianceAsCode/content, --fetch-remote-resources
+    test: ./smoke.sh ${TMT_PLAN_DATA}/arf_fetch-remote-resources-latest.xml
+
+
+tag:
+  - integration
+tier: 1

tests/smoke.sh

@@ -1,10 +1,17 @@
 #!/usr/bin/env bash
 # Test of the basic function
+# Usage: ./smoke.sh path_to_arf
+#   path_to_arf     (Default: ./test_data/arf-report.xml) Path where is tested ARF file
 
 set -e -o pipefail
 
+path_to_arf=$1
+if [ "$path_to_arf" = "" ]; then
+    path_to_arf="./test_data/arf-report.xml"
+fi
+
 # Generate report
-oscap-report < ./test_data/arf-report.xml > report.html
+oscap-report < "${path_to_arf}" > report.html
 
 # Search for some rule ID in the report
 grep -q "xccdf_org\.ssgproject\.content_rule_enable_fips_mode" report.html