Extend GitLab CI with build and test jobs for sanitizers (#2174)

Add a build and test job for each of ASAN, MSAN, TSAN, and UBSAN to the
GitLab pipeline such that current vulnerabilities will be more easily
visible and on each new commit, we can ensure that there are no
additional errors introduced. Furthermore, sanitizer test runs are run
separate from the existing mysql-test-run to isolate sanitizer error
from functional errors.

All new code of the whole pull request, including one or several files
that are either new files or modified ones, are contributed under the
BSD-new license. I am contributing on behalf of my employer
Amazon Web Services, Inc.

.gitlab-ci.yml

@@ -172,6 +172,39 @@ fedora-clang:
       - dependencies.dot
       - dependencies.png
 
+fedora-sanitizer:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel clang
+    - yum install -y /usr/lib64/libasan.so.6.0.0 /usr/lib64/libtsan.so.0.0.0 /usr/lib64/libubsan.so.1.0.0
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - export CXX=${CXX:-clang++}
+    - export CC=${CC:-clang}
+    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+    - export CFLAGS='-Wno-unused-command-line-argument'
+    - export CXXFLAGS='-Wno-unused-command-line-argument'
+    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS $SANITIZER .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: the build will fail consistently at 24% when trying to make using eatmydata
+    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+  parallel:
+    matrix:
+      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES, -DWITH_MSAN=YES]
+
 centos8:
   stage: build
   image: quay.io/centos/centos:stream8 # CentOS 8 is deprecated, use this Stream8 instead
@@ -246,10 +279,8 @@ centos7:
       - rpm
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
 
-mysql-test-run:
+.mysql-test-run: &mysql-test-run-def
   stage: test
-  dependencies:
-    - fedora
   script:
     # Install packages so tests and the dependencies install
     # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
@@ -265,7 +296,76 @@ mysql-test-run:
       main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
       main.mysql_upgrade_noengine : upgrade output order does not match the expected
       " > skiplist
-    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
+    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
+
+mysql-test-run:
+  stage: test
+  dependencies:
+    - fedora
+  <<: *mysql-test-run-def
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+# Duplicate of the above jobs, except we use sanitizer build jobs as a dependency. This is so we can keep
+# sanitizer errors separate from functional test failures. Currently, there is no way to run the same
+# job for different dependencies.
+#
+# Additionally, for each sanitizer MTR job, we enable --force-restart so that 
+# sanitizer errors can be traced to individual tests. The difference in test 
+# suite runtime as a result of this flag is negligable (~30s for the entire test suite).
+# (see https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_MYSQL_TEST_RUN_PL.html)
+mysql-test-run-asan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+  <<: *mysql-test-run-def
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-tsan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-ubsan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-msan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_MSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
   artifacts:
     when: always  # Also show results when tests fail
     reports: