diff --git a/.branches.gitlab-ci.yml b/.branches.gitlab-ci.yml new file mode 100644 index 000000000000..796ee8638f29 --- /dev/null +++ b/.branches.gitlab-ci.yml @@ -0,0 +1,161 @@ +stages: + - test + - perf + - cleanup + +variables: + PACK_RULE: package + +.develope_only_template: + only: + - branches + - schedules + - external_pull_requests + - merge_requests + +.full_only_template: + only: + - /^.*-full-ci$/ + - schedules + - external_pull_requests + - merge_requests + +.pack_only_template: + extends: .full_only_template + +.perf_only_template: + only: + - /^.*-full-ci-perf$/ + - /^.*-perf$/ + - external_pull_requests + - merge_requests + except: + - schedules + +# Jobs templates + +include: + - '.templates.gitlab-ci.yml' + - '.perf.gitlab-ci.yml' + +# Tests release and debug + +release: + extends: + - .develope_only_template + - .test_release_template + +debug: + extends: + - .develope_only_template + - .test_debug_template + +# Clang builds (Clang/Clang+LTO/Clang8+LTO/Clang8+ASAN) + +release_clang: + extends: + - .develope_only_template + - .test_release_clang_template + +release_lto: + extends: + - .full_only_template + - .test_release_lto_template + +release_lto_clang8: + extends: + - .full_only_template + - .test_release_lto_clang8_template + +release_asan_clang8: + extends: + - .develope_only_template + - .test_release_asan_clang8_template + +# Static builds w/ & w/o dockerfile + +static_build: + extends: + - .develope_only_template + - .test_static_build_template + +static_docker_build: + extends: + - .full_only_template + - .test_static_docker_build_template + +# OSX builds (14/15/15+LTO) + +osx_14_release: + extends: + - .full_only_template + - .test_osx_14_release_template + +osx_15_release: + extends: + - .develope_only_template + - .test_osx_15_release_template + +osx_15_release_lto: + extends: + - .full_only_template + - .test_osx_15_release_lto_template + +# FreeBSD build + +freebsd_12_release: + extends: + - .develope_only_template + - .test_freebsd_12_release_template + +# Build only packages and sources + +sources: + extends: .pack_template + script: + - ${GITLAB_MAKE} source + +centos_6: + extends: .test_centos_6_template + +centos_7: + extends: .test_centos_7_template + +centos_8: + extends: .test_centos_8_template + +fedora_28: + extends: .test_fedora_28_template + +fedora_29: + extends: .test_fedora_29_template + +fedora_30: + extends: .test_fedora_30_template + +fedora_31: + extends: .test_fedora_31_template + +ubuntu_14_04: + extends: .test_ubuntu_14_04_template + +ubuntu_16_04: + extends: .test_ubuntu_16_04_template + +ubuntu_18_04: + extends: .test_ubuntu_18_04_template + +ubuntu_19_10: + extends: .test_ubuntu_19_10_template + +ubuntu_20_04: + extends: .test_ubuntu_20_04_template + +debian_8: + extends: .test_debian_8_template + +debian_9: + extends: .test_debian_9_template + +debian_10: + extends: .test_debian_10_template diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7705631dd453..21b34f1744a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,502 +4,146 @@ stages: - cleanup variables: - GITLAB_MAKE: "make -f .gitlab.mk" + PACK_RULE: deploy -# Jobs templates - -.release_only_template: &release_only_definition - only: - - master - - /^.*-full-ci$/ - -.deploy_only_template: &deploy_only_definition +.release_only_template: only: - master - except: - schedules - - external_pull_requests - - merge_requests -.pack_only_template: &pack_only_definition - only: - - schedules - - external_pull_requests - - merge_requests - - /^.*-full-ci$/ +.pack_only_template: + extends: .release_only_template -.perf_only_template: &perf_only_definition +.perf_only_template: only: - master - - /^.*-perf$/ except: - schedules - variables: &perf_vars_definition - IMAGE_PERF: "${CI_REGISTRY}/${CI_PROJECT_PATH}/perf/ubuntu-bionic:perf_master" - IMAGE_PERF_BUILT: "${CI_REGISTRY}/${CI_PROJECT_PATH}/perf_tmp/ubuntu-bionic:perf_${CI_COMMIT_SHORT_SHA}" - -.docker_test_template: &docker_test_definition - image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-stretch:latest" - stage: test - tags: - - docker_test - -.docker_test_clang8_template: &docker_test_clang8_definition - image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest" - stage: test - tags: - - docker_test - -.pack_template: &pack_definition - <<: *pack_only_definition - stage: test - tags: - - deploy - script: - - ${GITLAB_MAKE} package -.pack_test_template: &pack_test_definition - <<: *pack_only_definition - stage: test - tags: - - deploy_test - script: - - ${GITLAB_MAKE} package - -.deploy_template: &deploy_definition - <<: *deploy_only_definition - stage: test - tags: - - deploy - script: - - ${GITLAB_MAKE} deploy - -.deploy_test_template: &deploy_test_definition - <<: *deploy_only_definition - stage: test - tags: - - deploy_test - script: - - ${GITLAB_MAKE} deploy - -.vbox_template: &vbox_definition - stage: test - before_script: - - ${GITLAB_MAKE} vms_start - after_script: - - ${GITLAB_MAKE} vms_shutdown - -.perf_docker_test_template: &perf_docker_test_definition - <<: *perf_only_definition - image: ${IMAGE_PERF_BUILT} - stage: perf - artifacts: - when: always - paths: - - "*_result.txt" - - "*_t_version.txt" - script: - - ${GITLAB_MAKE} perf_run +# Jobs templates -.perf_cleanup_definition: &perf_cleanup_definition - <<: *perf_only_definition - stage: cleanup - script: - - ${GITLAB_MAKE} perf_cleanup +include: + - '.templates.gitlab-ci.yml' + - '.perf.gitlab-ci.yml' -# Tests +# Tests release and debug release: - <<: *docker_test_definition - script: - - ${GITLAB_MAKE} test_debian_no_deps + extends: + - .release_only_template + - .test_release_template debug: - <<: *docker_test_definition - script: - - ${GITLAB_MAKE} test_coverage_debian_no_deps + extends: + - .release_only_template + - .test_debug_template + +# Clang builds (Clang/Clang+LTO/Clang8+LTO/Clang8+ASAN) release_clang: - <<: *docker_test_definition - variables: - CC: clang - CXX: clang++ - script: - - ${GITLAB_MAKE} test_debian_no_deps + extends: + - .release_only_template + - .test_release_clang_template release_lto: - <<: *release_only_definition - <<: *docker_test_clang8_definition - variables: - CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON - script: - - ${GITLAB_MAKE} test_debian_no_deps + extends: + - .release_only_template + - .test_release_lto_template release_lto_clang8: - <<: *release_only_definition - <<: *docker_test_clang8_definition - variables: - CC: clang-8 - CXX: clang++-8 - CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON - script: - - ${GITLAB_MAKE} test_debian_no_deps + extends: + - .release_only_template + - .test_release_lto_clang8_template release_asan_clang8: - <<: *docker_test_clang8_definition - script: - - ${GITLAB_MAKE} test_asan_debian_no_deps + extends: + - .release_only_template + - .test_release_asan_clang8_template + +# Static builds w/ & w/o dockerfile + +static_build: + extends: + - .release_only_template + - .test_static_build_template + +static_docker_build: + extends: + - .release_only_template + - .test_static_docker_build_template + +# OSX builds (14/15/15+LTO) osx_14_release: - <<: *release_only_definition - stage: test - tags: - - osx_14 - script: - - ${GITLAB_MAKE} test_osx + extends: + - .release_only_template + - .test_osx_14_release_template osx_15_release: - stage: test - tags: - - osx_15 - script: - - ${GITLAB_MAKE} test_osx + extends: + - .release_only_template + - .test_osx_15_release_template osx_15_release_lto: - <<: *release_only_definition - stage: test - tags: - - osx_15 - variables: - EXTRA_ENV: 'export CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON ;' - script: - - ${GITLAB_MAKE} test_osx - -freebsd_12_release: - <<: *vbox_definition - tags: - - vms_freebsd_12 - variables: - VMS_NAME: 'freebsd_12' - VMS_USER: 'vagrant' - VMS_PORT: '2232' - MAKE: 'gmake' - script: - - ${GITLAB_MAKE} vms_test_freebsd + extends: + - .release_only_template + - .test_osx_15_release_lto_template -# #### -# Perf -# #### +# FreeBSD build -# Pre-testing part +freebsd_12_release: + extends: + - .release_only_template + - .test_freebsd_12_release_template -perf_bootstrap: - <<: *perf_only_definition - stage: test - tags: - - deploy - script: - - ${GITLAB_MAKE} perf_prepare - after_script: - - ${GITLAB_MAKE} perf_cleanup_image - -# Testing part - -perf_sysbench: - <<: *perf_docker_test_definition - tags: - - docker_sh3_perf - variables: - <<: *perf_vars_definition - BENCH: 'sysbench' - -perf_tpcc: - <<: *perf_docker_test_definition - tags: - - docker_sh3_perf - variables: - <<: *perf_vars_definition - BENCH: 'tpcc' - -perf_ycsb_hash: - <<: *perf_docker_test_definition - tags: - - docker_sh2_perf - variables: - <<: *perf_vars_definition - BENCH: 'ycsb' - ARG: 'hash' - -perf_ycsb_tree: - <<: *perf_docker_test_definition - tags: - - docker_sh2_perf - variables: - <<: *perf_vars_definition - BENCH: 'ycsb' - ARG: 'tree' - -perf_nosqlbench_hash: - <<: *perf_docker_test_definition - tags: - - docker_sh1_perf - variables: - <<: *perf_vars_definition - BENCH: 'nosqlbench' - ARG: 'hash' - -perf_nosqlbench_tree: - <<: *perf_docker_test_definition - tags: - - docker_sh1_perf - variables: - <<: *perf_vars_definition - BENCH: 'nosqlbench' - ARG: 'tree' - -perf_cbench: - <<: *perf_docker_test_definition - tags: - - docker_sh2_perf - variables: - <<: *perf_vars_definition - BENCH: 'cbench' - -perf_linkbench_ssd: - <<: *perf_docker_test_definition - tags: - - docker_perf_ssd - variables: - <<: *perf_vars_definition - BENCH: 'linkbench' - -# Post-testing part - -remove_images_sh1: - <<: *perf_cleanup_definition - tags: - - sh1_shell - -remove_images_sh2: - <<: *perf_cleanup_definition - tags: - - sh2_shell - -remove_images_sh3: - <<: *perf_cleanup_definition - tags: - - sh3_shell - -remove_images_sh9: - <<: *perf_cleanup_definition - tags: - - sh9_shell - -# Packages and sources - -sources: - <<: *pack_definition - script: - - ${GITLAB_MAKE} source - -centos_6: - <<: *pack_definition - variables: - OS: 'el' - DIST: '6' - -centos_7: - <<: *pack_test_definition - variables: - OS: 'el' - DIST: '7' - -centos_8: - <<: *pack_test_definition - variables: - OS: 'el' - DIST: '8' - -fedora_28: - <<: *pack_test_definition - variables: - OS: 'fedora' - DIST: '28' - -fedora_29: - <<: *pack_test_definition - variables: - OS: 'fedora' - DIST: '29' - -fedora_30: - <<: *pack_test_definition - variables: - OS: 'fedora' - DIST: '30' - -fedora_31: - <<: *pack_test_definition - variables: - OS: 'fedora' - DIST: '31' - -ubuntu_14_04: - <<: *pack_definition - variables: - OS: 'ubuntu' - DIST: 'trusty' - -ubuntu_16_04: - <<: *pack_definition - variables: - OS: 'ubuntu' - DIST: 'xenial' - -ubuntu_18_04: - <<: *pack_definition - variables: - OS: 'ubuntu' - DIST: 'bionic' - -ubuntu_19_10: - <<: *pack_definition - variables: - OS: 'ubuntu' - DIST: 'eoan' - -ubuntu_20_04: - <<: *pack_definition - variables: - OS: 'ubuntu' - DIST: 'focal' - -debian_8: - <<: *pack_definition - variables: - OS: 'debian' - DIST: 'jessie' - -debian_9: - <<: *pack_definition - variables: - OS: 'debian' - DIST: 'stretch' - -debian_10: - <<: *pack_definition - variables: - OS: 'debian' - DIST: 'buster' - -# Deploy +# Build and deploy packages and sources sources_deploy: - <<: *deploy_definition + extends: .pack_template script: - ${GITLAB_MAKE} source_deploy centos_6_deploy: - <<: *deploy_definition - variables: - OS: 'el' - DIST: '6' + extends: .test_centos_6_template centos_7_deploy: - <<: *deploy_test_definition - variables: - OS: 'el' - DIST: '7' + extends: .test_centos_7_template centos_8_deploy: - <<: *deploy_test_definition - variables: - OS: 'el' - DIST: '8' + extends: .test_centos_8_template fedora_28_deploy: - <<: *deploy_test_definition - variables: - OS: 'fedora' - DIST: '28' + extends: .test_fedora_28_template fedora_29_deploy: - <<: *deploy_test_definition - variables: - OS: 'fedora' - DIST: '29' + extends: .test_fedora_29_template fedora_30_deploy: - <<: *deploy_test_definition - variables: - OS: 'fedora' - DIST: '30' + extends: .test_fedora_30_template fedora_31_deploy: - <<: *deploy_test_definition - variables: - OS: 'fedora' - DIST: '31' + extends: .test_fedora_31_template ubuntu_14_04_deploy: - <<: *deploy_definition - variables: - OS: 'ubuntu' - DIST: 'trusty' + extends: .test_ubuntu_14_04_template ubuntu_16_04_deploy: - <<: *deploy_definition - variables: - OS: 'ubuntu' - DIST: 'xenial' + extends: .test_ubuntu_16_04_template ubuntu_18_04_deploy: - <<: *deploy_definition - variables: - OS: 'ubuntu' - DIST: 'bionic' + extends: .test_ubuntu_18_04_template ubuntu_19_10_deploy: - <<: *deploy_definition - variables: - OS: 'ubuntu' - DIST: 'eoan' + extends: .test_ubuntu_19_10_template ubuntu_20_04_deploy: - <<: *deploy_definition - variables: - OS: 'ubuntu' - DIST: 'focal' + extends: .test_ubuntu_20_04_template debian_8_deploy: - <<: *deploy_definition - variables: - OS: 'debian' - DIST: 'jessie' + extends: .test_debian_8_template debian_9_deploy: - <<: *deploy_definition - variables: - OS: 'debian' - DIST: 'stretch' + extends: .test_debian_9_template debian_10_deploy: - <<: *deploy_definition - variables: - OS: 'debian' - DIST: 'buster' - -# Static builds - -static_build: - <<: *docker_test_definition - script: - - ${GITLAB_MAKE} test_static_build - -static_docker_build: - <<: *release_only_definition - stage: test - tags: - - deploy_test - script: - - ${GITLAB_MAKE} test_static_docker_build + extends: .test_debian_10_template diff --git a/.perf.gitlab-ci.yml b/.perf.gitlab-ci.yml new file mode 100644 index 000000000000..b4f132951231 --- /dev/null +++ b/.perf.gitlab-ci.yml @@ -0,0 +1,123 @@ +# ################ +# Performance only +# ################ + +# Jobs templates + +variables: + IMAGE_PERF: "${CI_REGISTRY}/${CI_PROJECT_PATH}/perf/ubuntu-bionic:perf_master" + IMAGE_PERF_BUILT: "${CI_REGISTRY}/${CI_PROJECT_PATH}/perf_tmp/ubuntu-bionic:perf_${CI_COMMIT_SHORT_SHA}" + +.perf_docker_test_template: + extends: .perf_only_template + image: ${IMAGE_PERF_BUILT} + stage: perf + artifacts: + when: always + paths: + - "*_result.txt" + - "*_t_version.txt" + script: + - ${GITLAB_MAKE} perf_run + +.perf_cleanup_definition: + extends: .perf_only_template + stage: cleanup + script: + - ${GITLAB_MAKE} perf_cleanup + +# Pre-testing part + +perf_bootstrap: + extends: .perf_only_template + stage: test + tags: + - deploy + script: + - ${GITLAB_MAKE} perf_prepare + after_script: + - ${GITLAB_MAKE} perf_cleanup_image + +# Testing part + +perf_sysbench: + extends: .perf_docker_test_template + tags: + - docker_sh3_perf + variables: + BENCH: 'sysbench' + +perf_tpcc: + extends: .perf_docker_test_template + tags: + - docker_sh3_perf + variables: + BENCH: 'tpcc' + +perf_ycsb_hash: + extends: .perf_docker_test_template + tags: + - docker_sh2_perf + variables: + BENCH: 'ycsb' + ARG: 'hash' + +perf_ycsb_tree: + extends: .perf_docker_test_template + tags: + - docker_sh2_perf + variables: + BENCH: 'ycsb' + ARG: 'tree' + +perf_nosqlbench_hash: + extends: .perf_docker_test_template + tags: + - docker_sh1_perf + variables: + BENCH: 'nosqlbench' + ARG: 'hash' + +perf_nosqlbench_tree: + extends: .perf_docker_test_template + tags: + - docker_sh1_perf + variables: + BENCH: 'nosqlbench' + ARG: 'tree' + +perf_cbench: + extends: .perf_docker_test_template + tags: + - docker_sh2_perf + variables: + BENCH: 'cbench' + +perf_linkbench_ssd: + extends: .perf_docker_test_template + tags: + - docker_perf_ssd + variables: + BENCH: 'linkbench' + +# Post-testing part + +remove_images_sh1: + extends: .perf_cleanup_definition + tags: + - sh1_shell + +remove_images_sh2: + extends: .perf_cleanup_definition + tags: + - sh2_shell + +remove_images_sh3: + extends: .perf_cleanup_definition + tags: + - sh3_shell + +remove_images_sh9: + extends: .perf_cleanup_definition + tags: + - sh9_shell diff --git a/.templates.gitlab-ci.yml b/.templates.gitlab-ci.yml new file mode 100644 index 000000000000..d93a778bf604 --- /dev/null +++ b/.templates.gitlab-ci.yml @@ -0,0 +1,227 @@ +variables: + GITLAB_MAKE: "make -f .gitlab.mk" + +# Jobs templates / helpers + +.docker_helper_test_template: + image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-stretch:latest" + stage: test + tags: + - docker_test + +.docker_helper_test_clang8_template: + image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest" + stage: test + tags: + - docker_test + +.vbox_helper_template: + stage: test + before_script: + - ${GITLAB_MAKE} vms_start + after_script: + - ${GITLAB_MAKE} vms_shutdown + +.pack_helper_template: + extends: .pack_only_template + stage: test + script: + - ${GITLAB_MAKE} ${PACK_RULE} + +.pack_template: + extends: .pack_helper_template + tags: + - deploy + +.pack_test_template: + extends: .pack_helper_template + tags: + - deploy_test + +# Tests release and debug + +.test_release_template: + extends: .docker_helper_test_template + script: + - ${GITLAB_MAKE} test_debian_no_deps + +.test_debug_template: + extends: .docker_helper_test_template + script: + - ${GITLAB_MAKE} test_coverage_debian_no_deps + +# Clang builds (Clang/Clang+LTO/Clang8+LTO/Clang8+ASAN) + +.test_release_clang_template: + extends: .docker_helper_test_template + variables: + CC: clang + CXX: clang++ + script: + - ${GITLAB_MAKE} test_debian_no_deps + +.test_release_lto_template: + extends: .docker_helper_test_clang8_template + variables: + CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON + script: + - ${GITLAB_MAKE} test_debian_no_deps + +.test_release_lto_clang8_template: + extends: .docker_helper_test_clang8_template + variables: + CC: clang-8 + CXX: clang++-8 + CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON + script: + - ${GITLAB_MAKE} test_debian_no_deps + +.test_release_asan_clang8_template: + extends: .docker_helper_test_clang8_template + script: + - ${GITLAB_MAKE} test_asan_debian_no_deps + +# Static builds w/ & w/o dockerfile + +.test_static_build_template: + extends: .docker_helper_test_template + script: + - ${GITLAB_MAKE} test_static_build + +.test_static_docker_build_template: + stage: test + tags: + - deploy_test + script: + - ${GITLAB_MAKE} test_static_docker_build + +# OSX builds (14/15/15+LTO) + +.test_osx_14_release_template: + stage: test + tags: + - osx_14 + script: + - ${GITLAB_MAKE} test_osx + +.test_osx_15_release_template: + stage: test + tags: + - osx_15 + script: + - ${GITLAB_MAKE} test_osx + +.test_osx_15_release_lto_template: + stage: test + tags: + - osx_15 + variables: + EXTRA_ENV: 'export CMAKE_EXTRA_PARAMS=-DENABLE_LTO=ON ;' + script: + - ${GITLAB_MAKE} test_osx + +# FreeBSD build + +.test_freebsd_12_release_template: + extends: .vbox_helper_template + tags: + - vms_freebsd_12 + variables: + VMS_NAME: 'freebsd_12' + VMS_USER: 'vagrant' + VMS_PORT: '2232' + MAKE: 'gmake' + script: + - ${GITLAB_MAKE} vms_test_freebsd + +# Packages/Deploy + +.test_centos_6_template: + extends: .pack_template + variables: + OS: 'el' + DIST: '6' + +.test_centos_7_template: + extends: .pack_test_template + variables: + OS: 'el' + DIST: '7' + +.test_centos_8_template: + extends: .pack_test_template + variables: + OS: 'el' + DIST: '8' + +.test_fedora_28_template: + extends: .pack_test_template + variables: + OS: 'fedora' + DIST: '28' + +.test_fedora_29_template: + extends: .pack_test_template + variables: + OS: 'fedora' + DIST: '29' + +.test_fedora_30_template: + extends: .pack_test_template + variables: + OS: 'fedora' + DIST: '30' + +.test_fedora_31_template: + extends: .pack_test_template + variables: + OS: 'fedora' + DIST: '31' + +.test_ubuntu_14_04_template: + extends: .pack_template + variables: + OS: 'ubuntu' + DIST: 'trusty' + +.test_ubuntu_16_04_template: + extends: .pack_template + variables: + OS: 'ubuntu' + DIST: 'xenial' + +.test_ubuntu_18_04_template: + extends: .pack_template + variables: + OS: 'ubuntu' + DIST: 'bionic' + +.test_ubuntu_19_10_template: + extends: .pack_template + variables: + OS: 'ubuntu' + DIST: 'eoan' + +.test_ubuntu_20_04_template: + extends: .pack_template + variables: + OS: 'ubuntu' + DIST: 'focal' + +.test_debian_8_template: + extends: .pack_template + variables: + OS: 'debian' + DIST: 'jessie' + +.test_debian_9_template: + extends: .pack_template + variables: + OS: 'debian' + DIST: 'stretch' + +.test_debian_10_template: + extends: .pack_template + variables: + OS: 'debian' + DIST: 'buster'