e2e.yml

#
# Copyright 2013-2024 Software Radio Systems Limited
#
# By using this file, you agree to the terms and conditions set
# forth in the LICENSE file which can be found at the top level of
# the distribution.
#

include:
  - project: softwareradiosystems/ci/tools
    ref: "17"
    file: .gitlab/ci-shared/setup/all.yml

variables:
  TESTBED:
    description: Retina Testbed Description
    options:
      - "zmq"
      - "zmq_deb"
      - "zmq_single_ue"
      - "zmq_4x4_mimo"
      - "zmq_srsue"
      - "rf_b200"
      - "rf_hp"
      - "android_b200"
      - "android_n300"
      - "android_x300"
      - "viavi"
      - "none"
    value: "zmq"
  MARKERS:
    description: Selected pytest marker (label / tag). Tests with that mark will run.
    value: ""
  KEYWORDS:
    description: Select tests by keyword expressions. This will run tests which contain names that match the given string expression (case-insensitive), which can include Python operators that use filenames, class names and function names as variables
    value: ""
  PYTEST_ARGS:
    description: Extra pytest args for the e2e suite to run
    value: ""
  RETINA_ARGS:
    description: Extra retina args for the e2e suite to run
    value: ""
  E2E_LOG_LEVEL:
    description: Log level to set for software under test in the e2e tests
    options:
      - debug
      - info
      - warning
      - error
    value: "info"

.txrx-lib: &txrx-lib
  - job: "build trx driver"
    optional: true
    artifacts: true
  - job: "build amariue zmq driver"
    optional: true
    artifacts: true

load retina variables:
  stage: ci
  rules:
    - if: $TESTBED == "none"
      when: never
    - if: $ON_MR
    - if: $CI_DESCRIPTION =~ /Nightly E2E Tests/
    - if: $ON_WEB
    - if: $ON_API
  script:
    - cat .gitlab/ci/e2e/.env
  artifacts:
    reports:
      dotenv: .gitlab/ci/e2e/.env

.prepare_test:
  variables:
    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
  before_script:
    - |
      eval K_PATH="\$$KUBECONFIG_VAR_NAME"
      export FORCE_COLOR=1
      export KUBECONFIG=$K_PATH
  needs: &retina-needs
    - job: "load retina variables"
      artifacts: true

e2e request validation:
  stage: static
  extends:
    - .prepare_test
  rules:
    - if: $ON_MR
      changes:
        - .gitlab/ci/e2e/**/*
  image:
    name: ${RETINA_REGISTRY_PREFIX}/launcher:${RETINA_VERSION}
    entrypoint: ["/bin/sh", "-c"]
  script:
    - |
      cd ${CI_PROJECT_DIR}/.gitlab/ci/e2e
      file_list=$(ls *.yml)
      for f in $file_list; do
          retina-request-checker --input $f
      done

.e2e-run:
  resource_group: e2e-${GROUP}
  timeout: 6h
  image:
    name: ${RETINA_REGISTRY_PREFIX}/launcher:${RETINA_VERSION}
    entrypoint: ["/bin/sh", "-c"]
  interruptible: false
  variables:
    ARTIFACT_COMPRESSION_LEVEL: "slowest"
    KUBERNETES_CPU_REQUEST: 2
    KUBERNETES_CPU_LIMIT: 2
    KUBERNETES_MEMORY_REQUEST: 2Gi
    KUBERNETES_MEMORY_LIMIT: 2Gi
    KUBERNETES_EPHEMERAL_STORAGE_REQUEST: "12G"
    KUBERNETES_EPHEMERAL_STORAGE_LIMIT: "12G"
    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
    GROUP: zmq
  tags:
    - "${RETINA_TAG}"
  extends:
    - .prepare_test
  rules:
    - if: $CI_DESCRIPTION =~ /Nightly E2E Tests/
  artifacts:
    paths:
      - tests/e2e/log
    when: always
    expire_in: 1 week
    reports:
      junit: tests/e2e/out.xml
  script:
    # Clean LFS files
    - |
      while read -r line; do
        git rm --cached "$line" >/dev/null
      done < <(git lfs ls-files | sed -r 's/^.{13}//')
      echo "srsGNB sources+build size is" $(du -hs .)
    # Remove any existing retina resource for this group
    - retina-delete-orchestration-network --user-name ^ci_${GROUP} --regex
    # Add extra secret env variables to the .env file
    - |
      echo "" >> .gitlab/ci/e2e/.env
      cat $RETINA_SECRET_ENV >> .gitlab/ci/e2e/.env
    # Set username for retina
    - |
      cd tests/e2e
      export LOGNAME=ci_${GROUP}_${GITLAB_USER_LOGIN}
    # Change sharing gnb file for complete src + build (With gnb) folder
    - |
      sed -i "s|- local_path: ../../build/apps/gnb/gnb|- local_path: ../../|; s|remote_path: /usr/local/bin|remote_path: $CI_PROJECT_DIR|; s|is_executable: true|is_executable: false|" ${CI_PROJECT_DIR}/.gitlab/ci/e2e/retina_request_${TESTBED}.yml
    # Run Retina
    - |
      E2E_CMD="retina-launcher --retina-request=${CI_PROJECT_DIR}/.gitlab/ci/e2e/retina_request_${TESTBED}.yml --log-folder=./log --html=./log/report.html --self-contained-html --junitxml=out.xml ${PYTEST_ARGS} -k '${KEYWORDS}' -m '${MARKERS}' --register-parameter ue.all.log_level=$E2E_LOG_LEVEL gnb.all.log_level=$E2E_LOG_LEVEL ${RETINA_ARGS}"
      echo "${E2E_CMD}"
      eval $E2E_CMD
  after_script:
    - |
      echo "*******************************************************************************************************************************"
      echo "Test report ---> https://softwareradiosystems.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/tests/e2e/log//report.html"
      echo "*******************************************************************************************************************************"
    - du -hs tests/e2e/log/

custom e2e:
  stage: manual
  extends: .e2e-run
  rules:
    - if: $TESTBED == "none"
      when: never
    - if: $ON_WEB
    - if: $ON_API
  needs:
    - *retina-needs
    - custom build
    - build trx driver

################################################################################
# ZMQ
################################################################################

smoke zmq:
  extends: .e2e-run
  stage: manual
  rules:
    - if: $ON_MR == "true" && $CI_MERGE_REQUEST_APPROVED == "true"
    - if: $ON_MR
      when: manual
      allow_failure: false
  interruptible: true
  variables:
    TESTBED: zmq
    MARKERS: "smoke"
    PYTEST_ARGS: "-x"
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.mac_enable=True gnb.all.rlc_enable=True"
    E2E_LOG_LEVEL: "debug"
  needs:
    - job: "smoke relwithdeb cached"
      artifacts: true
      optional: true
    - job: "smoke relwithdeb clean"
      artifacts: true
      optional: true
    - *txrx-lib
    - *retina-needs

.zmq:
  stage: zmq
  extends: .e2e-run
  variables:
    TESTBED: "zmq"
    E2E_LOG_LEVEL: "info"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs

srsue:
  extends: .zmq
  variables:
    TESTBED: "zmq_srsue"
    MARKERS: "zmq_srsue"
    E2E_LOG_LEVEL: "info"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 1UE:
  extends: .zmq
  variables:
    TESTBED: "zmq_single_ue"
    MARKERS: "zmq_single_ue"
    E2E_LOG_LEVEL: "info"
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.rlc_enable=True"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 1UE 4x4 mimo:
  extends: .zmq
  variables:
    TESTBED: "zmq_4x4_mimo"
    MARKERS: "zmq_4x4_mimo"
    E2E_LOG_LEVEL: "info"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 4UE deb:
  extends: .zmq
  variables:
    TESTBED: "zmq_deb"
    MARKERS: "smoke"
    E2E_LOG_LEVEL: "info"
  needs:
    - job: "basic package"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 32UE:
  extends: .zmq
  variables:
    MARKERS: "zmq and not smoke"
    E2E_LOG_LEVEL: "info"
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.rlc_enable=False"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs
  parallel:
    matrix:
      - KEYWORDS:
          [
            "attach_detach",
            "ping",
            "iperf and udp and band:3",
            "iperf and udp and not band:3",
            "iperf and tcp and band:3",
            "iperf and tcp and not band:3",
          ]

amari 32UE [reestablishment]:
  extends: .zmq
  variables:
    MARKERS: "zmq and not smoke"
    E2E_LOG_LEVEL: "info"
    KEYWORDS: reestablishment
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.mac_enable=True gnb.all.rlc_enable=False"
  allow_failure: true
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 32UE asan:
  extends: .zmq
  variables:
    MARKERS: "smoke"
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.mac_enable=True gnb.all.rlc_enable=True"
  needs:
    - job: "basic asan"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 32UE tsan:
  extends: .zmq
  variables:
    MARKERS: "smoke"
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.mac_enable=True gnb.all.rlc_enable=True"
  needs:
    - job: "basic tsan"
      artifacts: true
    - *txrx-lib
    - *retina-needs

amari 32UE memcheck:
  extends: .zmq
  variables:
    MARKERS: "zmq_valgrind"
    E2E_LOG_LEVEL: "warning"
    RETINA_ARGS: "gnb.all.pcap=True gnb.all.mac_enable=True gnb.all.rlc_enable=True"
  needs:
    - job: "basic memcheck"
      artifacts: true
    - *txrx-lib
    - *retina-needs

################################################################################
# TEST MODE
################################################################################

test mode ue:
  stage: zmq
  extends: .e2e-run
  variables:
    TESTBED: "test_mode"
    MARKERS: "test_mode"
    KEYWORDS: "test_ue"
    E2E_LOG_LEVEL: "warning"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ue tsan:
  extends: test mode ue
  when: manual
  allow_failure: true
  needs:
    - job: "basic tsan"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ue asan:
  extends: test mode ue
  needs:
    - job: "basic asan"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ue memcheck:
  extends: test mode ue
  needs:
    - job: "basic memcheck"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ru:
  stage: zmq
  extends: .e2e-run
  when: manual
  allow_failure: true
  variables:
    GROUP: "viavi"
    TESTBED: "viavi"
    MARKERS: "test_mode"
    KEYWORDS: "test_ru"
    E2E_LOG_LEVEL: "info"
  needs:
    - job: "basic avx512 dpdk"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ru tsan:
  extends: test mode ru
  variables:
    E2E_LOG_LEVEL: "warning"
  when: manual
  allow_failure: true
  needs:
    - job: "basic tsan"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ru asan:
  extends: test mode ru
  variables:
    E2E_LOG_LEVEL: "warning"
  when: manual
  allow_failure: true
  needs:
    - job: "basic asan"
      artifacts: true
    - *txrx-lib
    - *retina-needs

test mode ru memcheck:
  extends: test mode ru
  variables:
    E2E_LOG_LEVEL: "warning"
  when: manual
  allow_failure: true
  needs:
    - job: "basic memcheck"
      artifacts: true
    - *txrx-lib
    - *retina-needs

################################################################################
# RF
################################################################################

.rf:
  stage: rf
  extends: .e2e-run
  variables:
    GROUP: "rf"
    TESTBED: "rf_b200"
    MARKERS: "rf"
    E2E_LOG_LEVEL: "warning"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *retina-needs

amari b200 asan:
  extends: .rf
  variables:
    MARKERS: "rf_not_crash"
  needs:
    - job: "basic asan"
      artifacts: true
    - *retina-needs

validate b200 config:
  extends: .rf
  variables:
    MARKERS: "rf_b200"

validate n300 config:
  extends: .rf
  variables:
    MARKERS: "rf_n300"
    TESTBED: "android_n300"

################################################################################
# Android
###############################################################################

android b200:
  stage: rf
  extends: .e2e-run
  variables:
    GROUP: "rf"
    TESTBED: "android_b200"
    MARKERS: "android"
    E2E_LOG_LEVEL: "warning"
    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *retina-needs

android x300:
  stage: rf
  extends: .e2e-run
  variables:
    GROUP: "rf"
    TESTBED: "android_x300"
    MARKERS: "android_hp"
    E2E_LOG_LEVEL: "info"
    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
  needs:
    - job: "basic relwithdeb"
      artifacts: true
    - *retina-needs

android n300:
  extends: android x300
  variables:
    TESTBED: "android_n300"

################################################################################
# VIAVI
###############################################################################

viavi:
  stage: rf
  extends: .e2e-run
  variables:
    GROUP: "viavi"
    TESTBED: "viavi"
    MARKERS: "viavi"
    E2E_LOG_LEVEL: "warning"
    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
    RETINA_ARGS: "gnb.all.metrics_enable=True"
  needs:
    - job: "basic avx512 dpdk"
      artifacts: true
    - *retina-needs

################################################################################
# Garbage collector
################################################################################

.demolition:
  extends:
    - .prepare_test
  rules:
    - if: $CI_DESCRIPTION =~ /Nightly E2E Tests/
      when: always
  interruptible: false
  image:
    name: ${RETINA_REGISTRY_PREFIX}/launcher:${RETINA_VERSION}
    entrypoint: ["/bin/sh", "-c"]
  tags:
    - "${RETINA_TAG}"
  script:
    - |
      retina-garbage-collector --mode demolition

retina setup:
  stage: ci
  extends: .demolition

retina post:
  stage: .post
  extends: .demolition