config.yml

version: 2.1

references:
  - &OTP24 cimg/elixir:1.12  # Contains Erlang 24.0

executors:
  # OTP24
  otp_24:
    working_directory: ~/app
    docker:
      - image: *OTP24
  otp_24_redis:
    working_directory: ~/app
    docker:
      - image: *OTP24
      - image: circleci/redis:6.0.5
  otp_24_redis_minio_rmq:
    working_directory: ~/app
    docker:
      - image: *OTP24
      - image: circleci/redis:6.0.5
      - image: minio/minio:latest
        environment:
          - MINIO_ACCESS_KEY: "AKIAIAOAONIULXQGMOUA"
          - MINIO_SECRET_KEY: "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"
        command: ["server", "/data"]
      - image: rabbitmq:3.7-alpine

commands:
  install_erlang:
    parameters:
      otp_package:
        type: string
        description: Version of the Erlang package to install
    steps:
    - run:
        name: Install Erlang
        command: |
          sudo killall -9 apt-get || true && \
          sudo apt-get install -y esl-erlang=1:<<parameters.otp_package>>
  fetch_packages:
    steps:
    - run:
        name: Install basic packages
        command: |
          sudo killall -9 apt-get || true && \
          echo "Acquire::ForceIPv4 'true';" | sudo tee -a /etc/apt/apt.conf.d/99force-ipv4 && \
          wget https://packages.erlang-solutions.com/erlang-solutions_2.0_all.deb && \
          sudo dpkg -i erlang-solutions_2.0_all.deb && \
          sudo apt-get update && \
          sudo apt-get install unixodbc-dev -y && \
          sudo apt-get install unixodbc -y && \
          sudo apt-get install tdsodbc -y
  fetch_build_packages:
    steps:
      - run:
          name: Install packages necessary for building
          command: |
            sudo killall -9 apt-get || true && \
            echo "Acquire::ForceIPv4 'true';" | sudo tee -a /etc/apt/apt.conf.d/99force-ipv4 && \
            sudo apt-get update && \
            sudo apt-get install libssl-dev unixodbc-dev unixodbc tdsodbc rsync zlib1g-dev -y
  maybe_build_deps_and_cache:
    steps:
      - restore_cache:
          name: Maybe restore all rebar3 dependencies
          key: deps-cache--{{ checksum "rebar.lock" }}--{{ checksum "big_tests/rebar.lock" }}--{{ checksum "otp_version" }}
      - run:
          name: Get deps
          command: |
            tools/configure with-all
            ./rebar3 get-deps
      - save_cache:
          name: Cache downloaded and built dependencies
          key: deps-cache--{{ checksum "rebar.lock" }}--{{ checksum "big_tests/rebar.lock" }}--{{ checksum "otp_version" }}
          paths:
            - ~/.cache/rebar3
  maybe_build_certs:
    steps:
      - restore_cache:
          name: Restore certificates
          key: certs-cache
      - run:
          name: Maybe rebuild certificates if they're about to expire
          command: make certs
      - save_cache:
          name: Cache built certs
          key: certs-cache
          paths:
            - ~/app/tools/ssl
  build_and_cache_prod_build:
    steps:
      - run:
          name: Generate prod release
          command: |
            echo $ERLANG_VERSION > otp_version
            make rel
      - save_cache:
          key: prod-build-cache-{{ .Branch }}-{{ .Revision }}--{{ checksum "otp_version" }}
          paths: ~/app/_build/prod/rel/mongooseim
  restore_prod_build:
    steps:
      - restore_cache:
          key: prod-build-cache-{{ .Branch }}-{{ .Revision }}--{{ checksum "otp_version" }}
  persist_workspace:
    steps:
      - run: git gc
      - persist_to_workspace:
          root: ~/app/
          paths: ["*"]
  restore_workspace:
    steps:
      - attach_workspace: {at: ~/app}
      - prepare_for_cache
      - restore_cache:
          key: deps-cache--{{ checksum "rebar.lock" }}--{{ checksum "big_tests/rebar.lock" }}--{{ checksum "otp_version" }}
  install_dockerize:
    steps:
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
            rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
          environment:
            DOCKERIZE_VERSION: v0.6.1
  dockerize_wait:
    parameters: {port: {type: integer}}
    steps:
      - run:
          name: Wait for port << parameters.port >>
          command: dockerize -wait tcp://localhost:<< parameters.port >> -timeout 60s
  fetch_coverage_packages:
    steps:
      - run:
          name: Install pip3 and codecov packages
          command: |
            sudo killall -9 apt-get || true && \
            echo "Acquire::ForceIPv4 'true';" | sudo tee -a /etc/apt/apt.conf.d/99force-ipv4 && \
            sudo apt-get update && \
            sudo apt-get install python3-pip -y && \
            pip3 install codecov && codecov
  run_coverage_analysis:
    steps:
      - fetch_coverage_packages
      - run:
          name: Coverage
          when: on_success
          command: |
            echo "Success!"
            ./rebar3 codecov analyze
            codecov --disable=gcov --env PRESET
  upload_results_to_aws:
    steps:
      - run:
          name: Upload results
          when: always
          command: |
              tools/circleci-prepare-log-dir.sh
              if [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then tools/circleci-upload-to-s3.sh; fi
  publish_github_comment:
    steps:
      - run:
          name: Publish a comment to GitHub
          when: always
          command: |
              tools/circle-publish-github-comment.sh
  publish_github_summary_comment:
    steps:
      - run:
          name: Publish a summary comment to GitHub
          when: on_fail
          command: |
              tools/circle-publish-github-summary-comment.sh

  maybe_prepare_minio:
    steps:
      - run:
          name: Install minio client, prepare minio
          environment:
            - MINIO_HOSTNAME: "mongooseim-minio"
            - MINIO_ACCESS_KEY: "AKIAIAOAONIULXQGMOUA"
            - MINIO_SECRET_KEY: "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"
            - MINIO_BUCKET: "mybucket"
          command: |
            if [[ "$DB" == *minio* ]]; then
                curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ./mc
                chmod +x ./mc
                ./mc config host add ${MINIO_HOSTNAME} http://127.0.0.1:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
                ./mc mb ${MINIO_HOSTNAME}/${MINIO_BUCKET}
                ./mc policy set download "${MINIO_HOSTNAME}/${MINIO_BUCKET}"
            fi
  prepare_etc_hosts:
    steps:
      - run:
          name: Prepare /etc/hosts
          command: |
            echo '127.0.0.1 muc.localhost' | sudo tee -a /etc/hosts
            echo '127.0.0.1 localhost.bis' | sudo tee -a /etc/hosts
            echo '127.0.0.1 localhost' | sudo tee -a /etc/hosts
  restore_all_caches:
    steps:
      - restore_cache:
          name: Restore build artifacts
          key: build-cache-{{ .Branch }}-{{ .Revision }}--{{ checksum "otp_version" }}
      - restore_cache:
          name: Restore deps artifacts
          key: deps-cache--{{ checksum "rebar.lock" }}--{{ checksum "big_tests/rebar.lock" }}--{{ checksum "otp_version" }}
      - restore_cache:
          name: Restore certificates
          key: certs-cache
  prepare_for_cache:
    steps:
      - run:
          name: Prepare for cache
          command: echo $ERLANG_VERSION > otp_version

jobs:
  build:
    parallelism: 1
    machine:
      image: ubuntu-2004:202010-01
    parameters:
      otp_package:
        type: string
        description: Version of the Erlang package to install
    environment:
      SKIP_RELEASE: 1
      SKIP_COV: 0
    working_directory: ~/app
    steps:
      - checkout
      - fetch_packages
      - install_erlang:
          otp_package: <<parameters.otp_package>>
      - run:
          name: Prepare for cache
          command: |
            echo <<parameters.otp_package>> > otp_version
      - restore_cache:
          key: build-cache-{{ .Branch }}-{{ .Revision }}--{{ checksum "otp_version" }}
      - maybe_build_deps_and_cache
      - maybe_build_certs
      - run: ./rebar3 compile
      - run:
          name: Generate development releases
          command: ./tools/build-releases.sh
      - run:
          name: Build Big Tests
          command: |
            tools/build-tests.sh
      - save_cache:
          key: build-cache-{{ .Branch }}-{{ .Revision }}--{{ checksum "otp_version" }}
          paths: ./

  small_tests:
    parallelism: 1
    machine:
      image: ubuntu-2004:202010-01
    working_directory: ~/app
    parameters:
      otp_package:
        type: string
        description: Version of the Erlang package to install
    environment:
      PRESET: small_tests
    steps:
      - checkout
      - fetch_packages
      - install_erlang:
          otp_package: <<parameters.otp_package>>
      - run:
          name: Prepare for cache
          command: |
            echo <<parameters.otp_package>> > otp_version
      - restore_all_caches
      - run:
          name: Run Small Tests
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p small_tests -s true -e true
      - run_coverage_analysis
      - upload_results_to_aws
      - publish_github_comment

  big_tests:
    parallelism: 1
    machine:
      image: ubuntu-2004:202010-01
    working_directory: ~/app
    parameters:
      otp_package:
        type: string
        description: Version of the Erlang package to install
      preset:
        type: enum
        enum: [internal_mnesia, mysql_redis, odbc_mssql_mnesia, ldap_mnesia,
               elasticsearch_and_cassandra_mnesia, pgsql_mnesia, riak_mnesia]
        description: Preset to run
        default: internal_mnesia
      db:
        type: string
        description: Database to use
        default: mnesia
      tls_dist:
        type: boolean
        description: Erlang distribution with TLS enabled
        default: false
      spec:
        type: string
        description: Test spec file to use
        default: default.spec
    environment:
      MIX_ENV: test
      PRESET: <<parameters.preset>>
      DB: <<parameters.db>>
      TLS_DIST: <<parameters.tls_dist>>
      ELASTICSEARCH_VERSION: 5.6.9
      CASSANDRA_VERSION: 3.9
      TESTSPEC: <<parameters.spec>>
      REDIS_VERSION: 3.2.10
    steps:
      - checkout
      - fetch_packages
      - install_erlang:
          otp_package: <<parameters.otp_package>>
      - run:
          name: Prepare for cache
          command: |
            echo <<parameters.otp_package>> > otp_version
      - restore_all_caches
      - run:
          name: Setup database
          command: |
            tools/setup-db.sh
            if [ $PRESET = ldap_mnesia ]; then sudo tools/setup-ldap.sh; fi
            sudo tools/setup-rmq.sh; tools/setup-redis.sh
            echo '127.0.0.1 muc.localhost' | sudo tee -a /etc/hosts
      - run:
          name: Run Big Tests
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p $PRESET -s false
          no_output_timeout: 40m
      - run_coverage_analysis
      - run:
          name: Build Failed - Logs
          when: on_fail
          command: |
            echo "Failure!"
            if [ -s _build/mim1/rel/mongooseim/log/crash.log ]; then cat _build/mim1/rel/mongooseim/log/crash.log; fi
            if [ -s _build/mim2/rel/mongooseim/log/crash.log ]; then cat _build/mim2/rel/mongooseim/log/crash.log; fi
            tail -100 _build/mim1/rel/mongooseim/log/mongooseim.log.1
            tail -100 _build/mim2/rel/mongooseim/log/mongooseim.log.1
      - upload_results_to_aws
      - publish_github_comment
      - publish_github_summary_comment

  # #########################
  # jobs in docker containers
  docker_image:
    parallelism: 1
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
    steps:
      - checkout
      - prepare_for_cache
      - setup_remote_docker
      - restore_prod_build
      - run:
          name: Execute Docker image build and upload
          command: tools/circle-build-and-push-docker.sh

  docs_build_deploy:
    parallelism: 1
    docker:
      - image: cimg/python:3.9.0
    working_directory: ~/app
    steps:
      - checkout
      - run:
          name: Test that docs build
          command: |
            pip3 install mkdocs
            pip3 install mkdocs-material
            mkdocs build --strict
      - when:
          condition:
            or:
              - equal: [ master, << pipeline.git.branch >> ]
              - << pipeline.git.tag >>
          steps:
            - add_ssh_keys
            - run:
                name: Configure Git
                command: |
                  git config --global user.email "mongoose-im@erlang-solutions.com"
                  git config --global user.name "mongooseim"
            - run:
                name: Generate image tag
                command: |
                  set -x
                  if [ -n "$CIRCLE_TAG" ]; then
                      DOCS_TAG="$CIRCLE_TAG"
                  elif [ "$CIRCLE_BRANCH" == "master" ]; then
                      DOCS_TAG="latest"
                  else
                      DOCS_TAG=`echo $CIRCLE_BRANCH | sed 's,/,_,g'`
                  fi
                  env | sort
                  echo "export DOCS_TAG=$DOCS_TAG" >> $BASH_ENV
            - run:
                name: Build latest
                command: |
                  git fetch git@github.com:esl/MongooseDocs.git gh-pages:gh-pages
                  pip3 install mike
                  mike deploy $DOCS_TAG --remote git@github.com:esl/MongooseDocs.git --branch gh-pages --push --rebase

  build_in_docker:
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
      build_prod:
        type: boolean
        description: When enabled, prod release will be generated besides the dev ones
        default: false
    parallelism: 1
    environment:
      SKIP_RELEASE: 1
      SKIP_COV: 0
    steps:
      - checkout
      - prepare_for_cache
      - fetch_build_packages
      - maybe_build_deps_and_cache
      - maybe_build_certs
      - run: ./rebar3 compile
      - run:
          name: Generate development releases
          command: ./tools/build-releases.sh
      - when:
          condition: << parameters.build_prod >>
          steps: [ build_and_cache_prod_build ]
      - run:
          name: Build Big Tests
          command: tools/build-tests.sh
      - persist_workspace

  small_tests_in_docker:
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
    parallelism: 1
    environment:
      PRESET: small_tests
    steps:
      - restore_workspace
      - install_dockerize
      - dockerize_wait: {port: 6379}
      - run:
          name: Run Small Tests
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p small_tests -s true -e true
      - run_coverage_analysis
      - upload_results_to_aws
      - publish_github_comment

  big_tests_in_docker:
    executor: << parameters.executor >>
    parallelism: 1
    parameters:
      executor:
        type: executor
      preset:
        type: enum
        enum: [internal_mnesia, mysql_redis, odbc_mssql_mnesia, ldap_mnesia,
               elasticsearch_and_cassandra_mnesia, pgsql_mnesia, riak_mnesia]
        description: Preset to run
        default: internal_mnesia
      db:
        type: string
        description: Database to use
        default: mnesia
      tls_dist:
        type: boolean
        description: Erlang distribution with TLS enabled
        default: false
    environment:
      PRESET: <<parameters.preset>>
      DB: <<parameters.db>>
      TLS_DIST: <<parameters.tls_dist>>
      ELASTICSEARCH_VERSION: 5.6.9
      CASSANDRA_VERSION: 3.9
      REDIS_VERSION: 3.2.10
    steps:
      - restore_workspace
      - install_dockerize
      - dockerize_wait: {port: 6379} # Wait for redis
      - dockerize_wait: {port: 5672} # Wait for rmq
      - maybe_prepare_minio
      - prepare_etc_hosts
      - run:
          name: Run Big Tests
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p $PRESET -s false
          no_output_timeout: 40m
      - run_coverage_analysis
      - run:
          name: Build Failed - Logs
          when: on_fail
          command: |
            echo "Failure!"
            if [ -s _build/mim1/rel/mongooseim/log/crash.log ]; then cat _build/mim1/rel/mongooseim/log/crash.log; fi
            tail -100 _build/mim1/rel/mongooseim/log/mongooseim.log.1
      - upload_results_to_aws
      - publish_github_comment

  dialyzer:
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
    parallelism: 1
    steps:
      - restore_workspace
      - run:
          name: Run Dialyzer
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p dialyzer_only -s false

  xref:
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
    parallelism: 1
    steps:
      - restore_workspace
      - run:
          name: Run Xref
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p xref_only -s false
  edoc:
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
    parallelism: 1
    steps:
      - restore_workspace
      - run:
          name: Run Edoc
          command: |
            SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/test.sh -p edoc_only -s false

  package:
    parallelism: 1
    executor: << parameters.executor >>
    parameters:
      executor:
        type: executor
      platform:
        type: enum
        enum: [centos_7, debian_stretch]
        description: Platform type
      otp_package:
        type: string
        description: Version of the Erlang package to install
    environment:
      pkg_PLATFORM: <<parameters.platform>>
      ESL_ERLANG_PKG_VER: <<parameters.otp_package>>
      PRESET: pkg
    steps:
      - checkout
      - setup_remote_docker
      - run:
          name: Build package
          command: |
            KEEP_COVER_RUNNING=1 ./tools/test.sh -p pkg -s false

filters: &all_tags
  tags:
    only: /^\d+\.\d+\.\d+([a-z0-9\-\+])*/

workflows:
  version: 2
  build_and_test:
    jobs:
      # ============================================
      # ============= IN DOCKER EXECUTORS =============
      # ============= DOCUMENTATION =============
      - docs_build_deploy:
          name: build_and_deploy_docs
          context: mongooseim-org
          filters: *all_tags
      # ============= PACKAGES =============
      - package:
          name: centos_7
          executor: otp_24
          platform: centos_7
          context: mongooseim-org
          otp_package: 24.0-1
          filters: *all_tags
      - package:
          name: debian_stretch
          executor: otp_24
          platform: debian_stretch
          context: mongooseim-org
          otp_package: 24.0-1
          filters: *all_tags
      # ======== BASE DOCKER BUILDS ========
      - build_in_docker:
          name: otp_24_docker
          executor: otp_24
          context: mongooseim-org
          build_prod: true
      # ============= DIALYZER =============
      - dialyzer:
          name: dialyzer
          executor: otp_24
          context: mongooseim-org
          requires:
            - otp_24_docker
          filters: *all_tags
      # ============= XREF =============
      - xref:
          name: xref
          executor: otp_24
          context: mongooseim-org
          requires:
            - otp_24_docker
          filters: *all_tags
      # ============= EDOC =============
      - edoc:
          name: edoc
          executor: otp_24
          context: mongooseim-org
          requires:
            - otp_24_docker
          filters: *all_tags
      # ============= SMALL TESTS =============
      - small_tests_in_docker:
          name: small_tests_24
          executor: otp_24_redis
          context: mongooseim-org
          requires:
            - otp_24_docker
          filters: *all_tags
      # ============= BIG TESTS =============
      - big_tests_in_docker:
          name: internal_mnesia_24
          executor: otp_24_redis_minio_rmq
          context: mongooseim-org
          preset: internal_mnesia
          db: "mnesia minio"
          tls_dist: true
          requires:
            - otp_24_docker
          filters: *all_tags
      # ============================================
      # ============= IN VM EXECUTORS ==============
      # ============= TESTS OTP 23 =============
      - build:
          name: otp_23
          otp_package: 23.3.1-1
          context: mongooseim-org
          filters: *all_tags
      - small_tests:
          name: small_tests_23
          otp_package: 23.3.1-1
          context: mongooseim-org
          requires:
            - otp_23
          filters: *all_tags
      - big_tests:
          name: ldap_mnesia_23
          otp_package: 23.3.1-1
          preset: ldap_mnesia
          db: mnesia
          context: mongooseim-org
          requires:
            - otp_23
          filters: *all_tags
      - big_tests:
          name: pgsql_mnesia_23
          otp_package: 23.3.1-1
          preset: pgsql_mnesia
          db: pgsql
          context: mongooseim-org
          requires:
            - otp_23
          filters: *all_tags
      - big_tests:
          name: dynamic_domains_pgsql_mnesia_23
          otp_package: 23.3.1-1
          spec: dynamic_domains.spec
          preset: pgsql_mnesia
          db: pgsql
          context: mongooseim-org
          requires:
            - otp_23
          filters: *all_tags
      # ============= TESTS OTP 24 =============
      - build:
          name: otp_24
          otp_package: 24.0-1
          context: mongooseim-org
          filters: *all_tags
      - big_tests:
          name: ldap_mnesia_24
          otp_package: 24.0-1
          preset: ldap_mnesia
          db: mnesia
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: mysql_redis_24
          otp_package: 24.0-1
          preset: mysql_redis
          db: mysql
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: mssql_mnesia_24
          otp_package: 24.0-1
          preset: odbc_mssql_mnesia
          db: mssql
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: elasticsearch_and_cassandra_24
          otp_package: 24.0-1
          preset: elasticsearch_and_cassandra_mnesia
          db: "elasticsearch cassandra"
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: riak_mnesia_24
          otp_package: 24.0-1
          preset: riak_mnesia
          db: riak
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: pgsql_mnesia_24
          otp_package: 24.0-1
          preset: pgsql_mnesia
          db: pgsql
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: dynamic_domains_pgsql_mnesia_24
          otp_package: 24.0-1
          spec: dynamic_domains.spec
          preset: pgsql_mnesia
          db: pgsql
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: dynamic_domains_mysql_redis_24
          otp_package: 24.0-1
          spec: dynamic_domains.spec
          preset: mysql_redis
          db: mysql
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      - big_tests:
          name: dynamic_domains_mssql_mnesia_24
          otp_package: 24.0-1
          spec: dynamic_domains.spec
          preset: odbc_mssql_mnesia
          db: mssql
          context: mongooseim-org
          requires:
            - otp_24
          filters: *all_tags
      # ============= DOCKER IMAGE BUILD & UPLOAD =============
      - docker_image:
          name: docker_build_and_ship
          executor: otp_24
          context: mongooseim-org
          requires:
            - small_tests_23
            - ldap_mnesia_23
            - pgsql_mnesia_23
            - dynamic_domains_pgsql_mnesia_23

            - small_tests_24
            - internal_mnesia_24
            - pgsql_mnesia_24
            - mysql_redis_24
            - mssql_mnesia_24
            - ldap_mnesia_24
            - riak_mnesia_24
            - elasticsearch_and_cassandra_24
            - dynamic_domains_pgsql_mnesia_24
            - dynamic_domains_mysql_redis_24
            - dynamic_domains_mssql_mnesia_24

            - dialyzer
            - xref
            - edoc
          filters: *all_tags