Merge branch '4.9.0' into enhancement/23548-high-impact-vulnerabilities

.github/actions/indexer_connector_deps/action.yml

@@ -0,0 +1,53 @@
+name: "Indexer connector dependencies"
+description: "Download and compiles the dependencies for the indexer connector"
+
+runs:
+  using: "composite"
+  steps:
+      - name: Dependencies for local execution
+        if: env.ACT # Only run for local execution
+        shell: bash
+        run: |
+
+          # Obtain a copy of the signing key
+          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+
+          # Add the repository to your sources list
+          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
+
+          # Update packages
+          sudo apt-get update
+          sudo apt-get install -y cmake
+
+      - name: General dependencies
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libc6-dbg
+
+      - name: Build external deps
+        run: |
+          cd src
+          make deps TARGET=server -j2
+          make libwazuhext.so TARGET=server -j2
+        shell: bash
+
+      - name: Build http-request
+        run: |
+          cd src
+          SRC_FOLDER=$(pwd)
+
+          cd shared_modules/http-request
+          mkdir -p build && cd build
+          cmake .. -DCMAKE_PROJECT_NAME=http-request -DSRC_FOLDER=${SRC_FOLDER} && make -j2
+        shell: bash
+
+      - name: Build keystore
+        run: |
+          cd src
+          SRC_FOLDER=$(pwd)
+
+          cd shared_modules/keystore
+          mkdir -p build && cd build
+          cmake .. -DCMAKE_PROJECT_NAME=keystore -DSRC_FOLDER=${SRC_FOLDER} && make -j2
+        shell: bash

.github/actions/vulnerability_scanner_deps/action.yml

@@ -18,7 +18,7 @@ runs:
           # Update packages
           sudo apt-get update
           sudo apt-get install -y cmake
-      
+
       - name: General dependencies
         shell: bash
         run: |

.github/workflows/indexer-connector-tests.yml

@@ -0,0 +1,57 @@
+name: Indexer connector
+
+on:
+  workflow_dispatch:
+  pull_request:
+  # Pull request events
+    types: [synchronize, opened, reopened, ready_for_review]
+  # Path filtering
+    paths:
+      - ".github/workflows/-tests.yml"
+      - ".github/actions/compile_and_test/action.yml"
+      - ".github/actions/indexer_connector_deps/action.yml"
+      - "src/shared_modules/indexer_connector/**"
+      - "src/wazuh_modules/http-request/**"
+
+jobs:
+  indexer_connector-qa:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Project dependencies
+        uses: ./.github/actions/indexer_connector_deps
+
+      # indexer connector
+      - name: Indexer connector
+        uses: ./.github/actions/compile
+        with:
+          path: src/shared_modules/indexer_connector
+
+      # Install python dependencies
+      - name: Install dependencies
+        run: |
+          pip install -r src/shared_modules/indexer_connector/qa/requirements.txt
+
+      # Create folder for test logs
+      - name: Create folder for test logs
+        run: |
+          mkdir -p ${{ github.workspace }}/qa_logs
+
+      # Run indexer connector tests.
+      - name: Run tests
+        run: |
+          cd src
+          python -m pytest -vv shared_modules/indexer_connector/qa/ --log-cli-level=DEBUG
+          rm -rf tmp
+
+      # Upload log files of the tests
+      - name: Upload log files
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: QA log files
+          path: ${{ github.workspace }}/qa_logs

.github/workflows/integration-tests-aws-tier-0-1.yml

@@ -0,0 +1,182 @@
+name: Integration tests for AWS - Tier 0 and 1
+
+on:
+  workflow_dispatch:
+    inputs:
+      base_branch:
+        description: 'Base branch'
+        required: true
+        default: 'main'
+      base_qa_it_fw_branch:
+        description: 'Base qa-integration-framework branch'
+        required: true
+        default: 'main'
+  pull_request:
+    paths:
+      - ".github/workflows/integration-tests-aws-tier-0-1.yml"
+      - "wodles/aws/**"
+
+jobs:
+  build:
+    env:
+      BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+      BRANCH_BASE: ${{ github.base_ref || inputs.base_branch }}
+      QA_IT_FW_BRANCH: ${{ github.base_ref || inputs.base_qa_it_fw_branch }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.IT_AWS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.IT_AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: 'us-east-1'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version-file: ".github/workflows/.python-version-it"
+          architecture: x64
+      # Download and install integration tests framework.
+      - name: Download and install integration tests framework
+        run: |
+          if [ "X`git ls-remote https://github.com/wazuh/qa-integration-framework.git ${BRANCH_BASE}`" != "X" ]; then
+              QA_BRANCH=${BRANCH_BASE}
+          elif [ "X`git ls-remote https://github.com/wazuh/qa-integration-framework.git ${QA_IT_FW_BRANCH}`" != "X" ]; then
+              QA_BRANCH=${QA_IT_FW_BRANCH}
+          else
+              QA_BRANCH="main"
+          fi
+          git clone -b ${QA_BRANCH} --single-branch https://github.com/wazuh/qa-integration-framework.git
+          sudo pip install qa-integration-framework/
+          sudo rm -rf qa-integration-framework/
+      - name: Set AWS credentials file
+        run: |
+          sudo aws configure set aws_access_key_id ${{ secrets.IT_AWS_KEY_ID }} --profile default
+          sudo aws configure set aws_secret_access_key ${{ secrets.IT_AWS_SECRET_ACCESS_KEY }} --profile default
+          sudo aws configure set default.region ${AWS_DEFAULT_REGION} --profile default
+      # Build wazuh server for linux.
+      - name: Build wazuh server for linux
+        run: |
+          make deps -C src TARGET=server -j2
+          make -C src TARGET=server -j2
+      # Install wazuh server for linux.
+      - name: Install wazuh server for linux
+        run: |
+          echo 'USER_LANGUAGE="en"' > ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_NO_STOP="y"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_INSTALL_TYPE="server"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo "USER_DIR=/var/ossec" >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_EMAIL="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_SYSCHECK="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_ROOTCHECK="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_SYSCOLLECTOR="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_SCA="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_WHITE_LIST="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_SYSLOG="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_AUTHD="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_ENABLE_UPDATE_CHECK="n"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          echo 'USER_AUTO_START="y"' >> ./etc/preloaded-vars.conf
+          echo "" >> ./etc/preloaded-vars.conf
+          sudo sh install.sh
+          rm ./etc/preloaded-vars.conf
+      # Run AWS integration tests.
+      - name: Run Parser related tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/aws_s3.py') ||
+            contains(steps.get_modified_files.outputs.files, 'wodles/aws/aws_tools.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 test_aws/test_parser.py
+      - name: Run every test due to base WazuhIntegration class change or manual dispatch
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/wazuh_integration.py') ||
+            ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 test_aws/
+      # Bucket tests
+      - name: Run Custom Buckets tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k kms test_aws/
+          sudo python3 -m pytest --tier 0 --tier 1 -k macie test_aws/
+          sudo python3 -m pytest --tier 0 --tier 1 -k trusted_advisor test_aws/
+      - name: Run Config tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/config.py') ||
+            contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k config test_aws/
+      - name: Run GuardDuty tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/guardduty.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k guardduty test_aws/
+      - name: Run CloudTrail tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/cloudtrail.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k cloudtrail test_aws/
+      - name: Run Load Balancers tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/load_balancers.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k alb test_aws/
+          sudo python3 -m pytest --tier 0 --tier 1 -k clb test_aws/
+          sudo python3 -m pytest --tier 0 --tier 1 -k nlb test_aws/
+      - name: Run Server Access tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/server_access.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k server_access test_aws/
+      - name: Run Umbrella tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/umbrella.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k cisco test_aws/
+      - name: Run VPC Flow tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/vpcflow.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k vpc test_aws/
+      - name: Run WAF tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/waf.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/buckets_s3/aws_bucket.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k waf test_aws/
+      # Services tests
+      - name: Run CloudWatch tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/services/cloudwatchlogs.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/services/aws_service.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k cloudwatch test_aws/
+      - name: Run Inspector tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/services/inspector.py') ||
+          contains(steps.get_modified_files.outputs.files, 'wodles/aws/services/aws_service.py')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 -k inspector test_aws/
+      # Custom Logs Buckets tests
+      - name: Run Inspector tests
+        if: contains(steps.get_modified_files.outputs.files, 'wodles/aws/subscribers/**')
+        run: |
+          cd tests/integration
+          sudo python3 -m pytest --tier 0 --tier 1 test_aws/test_custom_bucket.py

.github/workflows/integration-tests-fim-tier-0-1-macos.yml

@@ -1,4 +1,4 @@
-name: Integration tests for fim on MacOS - Tier 0 and 1
+name: Integration tests for FIM on MacOS - Tier 0 and 1
 
 on:
   workflow_dispatch:

.github/workflows/packages-build-manager.yml

@@ -150,6 +150,9 @@ jobs:
       - name: Upload package to S3
         working-directory: packages
         run: |
-          for file in /tmp/*manager*; do
-            aws s3 cp $file s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/
-          done
+          aws s3 cp /tmp/*manager*.${{ inputs.system }} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/
+
+      - name: Upload checksum to S3
+        if: ${{ inputs.checksum == true }}
+        run: |
+          aws s3 cp /tmp/*manager*.${{ inputs.system }}.sha512 s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/