.gitlab-ci.yml

variables:

    SERVER_NAME: dotmesh-server
    PROVISIONER_NAME: dotmesh-dynamic-provisioner

    CI_DOCKER_REGISTRY: registry.containers.ovh.net/dotmesh
    RELEASE_DOCKER_REGISTRY: quay.io/dotmesh

    CI_DOCKER_TAG: $CI_COMMIT_SHA

    CI_DOCKER_SERVER_IMAGE: $CI_DOCKER_REGISTRY/dotmesh-server:$CI_COMMIT_SHA
    CI_DOCKER_PROVISIONER_IMAGE: $CI_DOCKER_REGISTRY/dotmesh-dynamic-provisioner:$CI_COMMIT_SHA
    CI_DOCKER_OPERATOR_IMAGE: $CI_DOCKER_REGISTRY/dotmesh-operator:$CI_COMMIT_SHA
    CI_DOCKER_DIND_PROVISIONER_IMAGE: $CI_DOCKER_REGISTRY/dind-dynamic-provisioner:$CI_COMMIT_SHA

    CI_SERVICE_BEING_TESTED: dotmesh

stages:
    - build
    - test
    - deploy
    - manual_deploy

before_script:
    - mkdir -p gopath
    - export GOPATH=$(pwd)/gopath
    - cd gopath
    - mkdir -p src/github.com/dotmesh-io
    - ln -s `realpath ..` ./src/github.com/dotmesh-io/dotmesh
    - cd src/github.com/dotmesh-io/dotmesh
    # ensure master and the given branch are up-to-date so that versioner works reliably
    - git checkout master && git pull --ff-only origin master
    - git checkout $CI_COMMIT_REF_NAME && git reset --hard $CI_COMMIT_SHA
    - while [ -f /dotmesh-test-cleanup.lock ]; do echo "waiting for test cleanup..."; sleep 1; done
    - docker login -u $QUAY_USER -p $QUAY_PASSWORD quay.io
    - docker login -u $OVH_REGISTRY_USERNAME -p $OVH_REGISTRY_PASSWORD registry.containers.ovh.net

build_client_linux:
    stage: build
    tags:
        - ubuntu
        - fast
    script:
        - (cd cmd/dm && ./rebuild.sh Linux)
    artifacts:
        paths:
            - binaries
        expire_in: 1 week

build_client_macos:
    stage: build
    tags:
        - ubuntu
        - fast
    script:
        - (cd cmd/dm && ./rebuild.sh Darwin)
    artifacts:
        paths:
            - binaries
        expire_in: 1 week

build_yaml:
    stage: build
    tags:
        - ubuntu
        - fast
    script:
        - (cd kubernetes && ./rebuild.sh)
    artifacts:
        paths:
            - yaml
        expire_in: 1 week

build_server:
    stage: build
    tags:
        - ubuntu
        - fast
    script:
        # push to the OVH registry.
        - (cd cmd/dotmesh-server && ./rebuild.sh) # also pushes
        - echo "pushed $CI_DOCKER_SERVER_IMAGE"
        - echo "pushed $CI_DOCKER_PROVISIONER_IMAGE"
        - echo "pushed $CI_DOCKER_OPERATOR_IMAGE"
        - echo "pushed $CI_DOCKER_DIND_PROVISIONER_IMAGE"
        # push to quay.io, by overriding various variables (although notably
        # not CI_DOCKER_DIND_PROVISIONER_IMAGE because that's test-only
        # tooling).
        - (cd cmd/dotmesh-server && export CI_DOCKER_SERVER_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-server:$CI_COMMIT_SHA; export CI_DOCKER_PROVISIONER_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-dynamic-provisioner:$CI_COMMIT_SHA; export CI_DOCKER_OPERATOR_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-operator:$CI_COMMIT_SHA; ./rebuild.sh; echo "pushed $CI_DOCKER_SERVER_IMAGE"; echo "pushed $CI_DOCKER_PROVISIONER_IMAGE"; echo "pushed $CI_DOCKER_OPERATOR_IMAGE")
    artifacts:
        paths:
            - cmd/dotmesh-server/target
        expire_in: 1 week

# run sanity check that things still work on Windows and macOS
#windows_docker_stable:
#    stage: test
#    tags:
#        - windows
#        - docker-stable
#    before_script:
#        - |
#          bash -c "
#          mkdir -p gopath
#          export GOPATH=$(pwd)/gopath
#          cd gopath
#          mkdir -p src/github.com/dotmesh-io
#          ln -s `realpath ..` ./src/github.com/dotmesh-io/dotmesh
#          cd src/github.com/dotmesh-io/dotmesh
#          "
#    script:
#        - |
#          bash -c "
#          cd cmd/dotmesh-server;
#          docker build -t dotmesh-server . || (sleep 30; docker build -t dotmesh-server .)
#          cd ../..
#          ./scripts/smoke.sh $(pwd)/binaries/Linux/dm dotmesh-server
#          docker system prune -fa
#          "

macos_docker_stable:
    stage: test
    dependencies:
        - build_server
        - build_client_macos
    tags:
        - macos
        - docker-stable
    script:
        - docker pull $CI_DOCKER_REGISTRY/dotmesh-server:$CI_DOCKER_TAG
        - ./scripts/smoke.sh $(pwd)/binaries/Darwin/dm dotmesh-server
        - docker version
        - which docker
        - docker container prune -f

macos_docker_edge:
    stage: test
    dependencies:
        - build_server
        - build_client_macos
    tags:
        - macos
        - docker-edge
    script:
        - docker pull $CI_DOCKER_REGISTRY/dotmesh-server:$CI_DOCKER_TAG
        - ./scripts/smoke.sh $(pwd)/binaries/Darwin/dm dotmesh-server
        - docker version
        - which docker
        - docker container prune -f

linux_smoke:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - docker pull $CI_DOCKER_REGISTRY/dotmesh-server:$CI_DOCKER_TAG
        - ./scripts/smoke.sh $(pwd)/binaries/Linux/dm dotmesh-server
        - docker version
        - which docker
        - docker container prune -f

versioner_unit_tests:
    stage: test
    tags:
        - ubuntu
        - fast
    script:
        - cd cmd/versioner && go test

# run full dind tests on linux
linux_default_dot:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestDefaultDot
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_single_node:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestSingleNode
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_two_nodes_same_cluster:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestTwoNodesSameCluster
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_kubernetes_volumes:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - pwd
        - ./test.sh -run TestKubernetesVolumes
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_deletion_simple:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestDeletionSimple
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_deletion_complex:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestDeletionComplex
    artifacts:
        paths:
        - extracted_logs
        when: always
    allow_failure: true

linux_backup_and_restore_two_single_node_clusters:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestBackupAndRestoreTwoSingleNodeClusters
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_two_single_node_clusters:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestTwoSingleNodeClusters
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_three_single_node_clusters:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestThreeSingleNodeClusters
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_two_double_node_clusters:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestTwoDoubleNodeClusters
    artifacts:
        paths:
        - extracted_logs
        when: always

# Test disabled while we rethink how it works (see comments in acceptance_test.go)
.linux_kubernetes_test_tooling:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -run TestKubernetesTestTooling
    artifacts:
        paths:
        - extracted_logs
        when: always

linux_stress:
    stage: test
    dependencies:
        - build_server
        - build_client_linux
    tags:
        - ubuntu
        - fast
    script:
        - ./test.sh -timeout 30m -run TestStress
    artifacts:
        paths:
        - extracted_logs
        when: always
    when: manual

trigger_e2e_test:
    stage: deploy
    tags:
        - ubuntu
        - fast
    script:
        - echo - curl -X POST -F "token=$E2E_TRIGGER_TOKEN" -F "ref=master" -F "variables[DOTMESH_CI_COMMIT_SHA]=$CI_COMMIT_SHA" -F "variables[DOTMESH_CI_BUILD_REF_NAME]=$CI_BUILD_REF_NAME" http://gitlab.dotmesh.io:9999/api/v4/projects/31/trigger/pipeline
        - curl -X POST -F "token=$E2E_TRIGGER_TOKEN" -F "ref=master" -F "variables[DOTMESH_CI_COMMIT_SHA]=$CI_COMMIT_SHA" -F "variables[DOTMESH_CI_BUILD_REF_NAME]=$CI_BUILD_REF_NAME" http://gitlab.dotmesh.io:9999/api/v4/projects/31/trigger/pipeline
    except:
        - /^release-.*$/

deploy_unstable_build:
    stage: deploy
    dependencies:
        - build_client_linux
        - build_client_macos
        - build_yaml
    tags:
        - ubuntu
        - fast
    script:
        - "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no releases@get.dotmesh.io mkdir -p /pool/releases/unstable/$CI_COMMIT_REF_NAME"
        - "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz binaries/{Linux,Darwin} yaml releases@get.dotmesh.io:/pool/releases/unstable/$CI_COMMIT_REF_NAME"
        - "echo \"Deployed to https://get.dotmesh.io/unstable/$CI_COMMIT_REF_NAME\""
    except:
        - /^release-.*$/

deploy_release_build:
    stage: deploy
    dependencies:
        - build_client_linux
        - build_client_macos
        - build_yaml
    tags:
        - ubuntu
        - fast
    script:
        - VERSION=`cd cmd/versioner; go run versioner.go`; echo VERSION=$VERSION
        - "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no releases@get.dotmesh.io mkdir -p /pool/releases/$VERSION"
        - "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz binaries/{Linux,Darwin} yaml releases@get.dotmesh.io:/pool/releases/$VERSION"
    only:
        - /^release-.*$/

mark_release_as_stable:
  stage: manual_deploy
  script:
    - VERSION=`cd cmd/versioner; go run versioner.go`
    - "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no releases@get.dotmesh.io sh -c \"true && rm -f /pool/releases/Linux && ln -s $VERSION/Linux /pool/releases && rm -f /pool/releases/Darwin && ln -s $VERSION/Darwin /pool/releases && rm -f /pool/releases/yaml && ln -fs $VERSION/yaml /pool/releases\""
  only:
    - /^release-.*$/
  when: manual