deploy.yml

# Copyright Contributors to the Packit project.
# SPDX-License-Identifier: MIT

---
- name: Packit Service deployment
  hosts: all
  # feel free to override these in vars/
  vars:
    validate_certs: true
    service: "{{ lookup('env', 'SERVICE') | default('packit', True) }}"
    deployment: "{{ lookup('env', 'DEPLOYMENT') }}"
    sandbox_namespace: "{{ service }}-{{ deployment }}-sandbox"
    with_tokman: true
    with_fedmsg: true
    with_redis_commander: false
    with_flower: false
    with_dashboard: true
    with_beat: true
    with_pushgateway: true
    with_repository_cache: true
    repository_cache_storage: 16Gi
    with_sandbox: true
    push_dev_images: false
    image: quay.io/packit/packit-service:{{ deployment }}
    image_worker: quay.io/packit/packit-worker:{{ deployment }}
    image_fedmsg: quay.io/packit/packit-service-fedmsg:{{ deployment }}
    image_dashboard: quay.io/packit/dashboard:{{ deployment }}
    image_tokman: quay.io/packit/tokman:{{ deployment }}
    # project_dir is set in tasks/project-dir.yml
    path_to_secrets: "{{ project_dir }}/secrets/{{ service }}/{{ deployment }}"
    # to be used in Image streams as importPolicy:scheduled value
    auto_import_images: "{{(deployment != 'prod')}}"
    # used in dev/zuul deployment to tag & push images to cluster
    # https://github.com/packit/deployment/issues/112#issuecomment-673343049
    # container_engine: "{{ lookup('pipe', 'command -v podman 2> /dev/null || echo docker') }}"
    container_engine: docker
    celery_app: packit_service.worker.tasks
    celery_retry_limit: 2
    celery_retry_backoff: 3
    workers_all_tasks: 1
    workers_short_running: 0
    workers_long_running: 0
    distgit_url: https://src.fedoraproject.org/
    distgit_namespace: rpms
    pushgateway_address: http://pushgateway
    # Check that the deployment repo is up-to-date
    check_up_to_date: true
    # Check that the current vars file is up-to-date with the template
    check_vars_template_diff: true
    deployment_repo_url: https://github.com/packit/deployment.git
    # used by a few tasks below
    k8s_apply: false
  tasks:
    - include_tasks: tasks/project-dir.yml
      tags:
        - always

    - name: include variables
      ansible.builtin.include_vars: "{{ project_dir }}/vars/{{ service }}/{{ deployment }}.yml"
      tags:
        - always

    - include_tasks: tasks/check-up-to-date.yml
      tags:
        - always

    - name: Getting deploymentconfigs
      include_tasks: tasks/set-facts.yml
      tags:
        - always

      # to be able to read the github_app_id from the configuration file in tokman
    - name: include packit-service configuration
      ansible.builtin.include_vars:
        file: "{{ path_to_secrets }}/packit-service.yaml"
        name: packit_service_config
      tags:
        - tokman

    - name: include extra secret vars
      ansible.builtin.include_vars: "{{ path_to_secrets }}/extra-vars.yml"
      tags:
        - always

    - name: Get k8s token and check it.
      block:
        - name: get kubeconfig token
          ansible.builtin.command: oc whoami -t
          register: kubeconfig_token
          changed_when: false
        - name: check if tokens match
          ansible.builtin.assert:
            that:
              - kubeconfig_token.stdout == api_key
            msg: "OpenShift API token defined in vars/ does not match token from your current environment."
      tags:
        - always

    - name: Push dev images to local registry
      block:
        - name: Set --tls-verify=false if podman is used
          ansible.builtin.set_fact: tls_verify_false="{{ '--tls-verify=false' if 'podman' in container_engine else '' }}"
          changed_when: false
        - name: Login to local cluster
          ansible.builtin.shell: "{{ container_engine }} login -u developer -p $(oc whoami -t) 172.30.1.1:5000 {{ tls_verify_false }}"
          changed_when: false
        - name: Inspect service image
          ansible.builtin.command: "{{ container_engine }} inspect {{ image }}"
          changed_when: false
        - name: Tag the image with :dev
          ansible.builtin.command: "{{ container_engine }} tag {{ image }} 172.30.1.1:5000/myproject/packit-service:dev"
          changed_when: true
        - name: Push the image
          ansible.builtin.command: "{{ container_engine }} push 172.30.1.1:5000/myproject/packit-service:dev {{ tls_verify_false }}"
          changed_when: true
        - name: Inspect worker image
          ansible.builtin.command: "{{ container_engine }} inspect {{ image_worker }}"
          changed_when: false
        - name: Tag the image with :dev
          ansible.builtin.command: "{{ container_engine }} tag {{ image_worker }} 172.30.1.1:5000/myproject/packit-worker:dev"
          changed_when: true
        - name: Push the image
          ansible.builtin.command: "{{ container_engine }} push 172.30.1.1:5000/myproject/packit-worker:dev {{ tls_verify_false }}"
          changed_when: true
      when: push_dev_images
      tags:
        - packit-service
        - packit-worker
        - packit-service-beat

    - name: Deploy secrets
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/secret-packit-ssh.yml.j2') }}"
        - "{{ lookup('template', '{{ project_dir }}/openshift/secret-packit-secrets.yml.j2') }}"
        - "{{ lookup('template', '{{ project_dir }}/openshift/secret-packit-config.yml.j2') }}"
        - "{{ lookup('template', '{{ project_dir }}/openshift/secret-sentry.yml.j2') }}"
        - "{{ lookup('template', '{{ project_dir }}/openshift/secret-postgres.yml.j2') }}"
        - "{{ lookup('template', '{{ project_dir }}/openshift/secret-aws.yml.j2') }}"
      tags:
        - secrets

    - name: Set up sandbox namespace
      block:
        - name: Set finalizers to sandbox namespace
          include_tasks: tasks/k8s.yml
          loop:
            - "{{ lookup('template', '{{ project_dir }}/openshift/sandbox-namespace.yml.j2') }}"
        - name: Add edit role to service account in sandbox namespace
          ansible.builtin.command: oc adm -n {{ sandbox_namespace }} policy add-role-to-user edit system:serviceaccount:{{ project }}:default
          register: rolebinding
          changed_when: "'added:' in rolebinding.stdout"
      when: with_sandbox

    - name: Deploy postgres
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/postgres.yml.j2')  }}"
      tags:
        - postgres

    - name: Deploy redis
      vars:
        k8s_apply: true
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('file', '{{ project_dir }}/openshift/redis.yml') }}"
      tags:
        - redis

    - name: Deploy packit-service
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/packit-service.yml.j2')  }}"
      tags:
        - packit-service

    - name: Deploy repository cache PVCs for packit-workers that serves both queues
      vars:
        name: "packit-worker-{{ item }}"
      k8s:
        namespace: "{{ sandbox_namespace }}"
        definition: "{{ lookup('template', '{{ project_dir }}/openshift/sandcastle-volumes-for-cache.yml.j2') }}"
        host: "{{ host }}"
        api_key: "{{ api_key }}"
        validate_certs: "{{ validate_certs }}"
      loop: "{{ range(0, workers_all_tasks) | list }}"
      tags:
        - packit-worker
      when: workers_all_tasks > 0 and with_repository_cache

    - name: Deploy packit-worker to serve both queues
      vars:
        name: packit-worker
        queues: "short-running,long-running"
        worker_replicas: "{{ workers_all_tasks }}"
        # cloning repos is memory intensive: glibc needs 300M+, kernel 600M+
        # during cloning, we need to account for git and celery worker processes
        worker_requests_memory: "384Mi"
        worker_requests_cpu: "100m"
        worker_limits_memory: "1024Mi"
        worker_limits_cpu: "400m"
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}"
      tags:
        - packit-worker
      when: workers_all_tasks > 0

    - name: Deploy packit-worker to serve short-running queue
      vars:
        name: packit-worker-short-running
        queues: "short-running"
        worker_replicas: "{{ workers_short_running }}"
        # Short-running tasks are just interactions with different services.
        # They should not require a lot of memory/cpu.
        worker_requests_memory: "320Mi"
        worker_requests_cpu: "100m"
        worker_limits_memory: "320Mi"
        worker_limits_cpu: "100m"
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}"
      tags:
        - packit-worker
      when: workers_short_running > 0

    - name: Deploy repository cache PVCs for packit-workers that serves long-running queue
      vars:
        name: "packit-worker-long-running-{{ item }}"
      k8s:
        namespace: "{{ sandbox_namespace }}"
        definition: "{{ lookup('template', '{{ project_dir }}/openshift/sandcastle-volumes-for-cache.yml.j2') }}"
        host: "{{ host }}"
        api_key: "{{ api_key }}"
        validate_certs: "{{ validate_certs }}"
      loop: "{{ range(0, workers_long_running) | list }}"
      tags:
        - packit-worker
      when: workers_long_running > 0 and with_repository_cache

    - name: Deploy packit-worker to serve long-running queue
      vars:
        name: packit-worker-long-running
        queues: "long-running"
        worker_replicas: "{{ workers_long_running }}"
        # cloning repos is memory intensive: glibc needs 300M+, kernel 600M+
        # during cloning, we need to account for git and celery worker processes
        worker_requests_memory: "384Mi"
        worker_requests_cpu: "100m"
        worker_limits_memory: "1024Mi"
        worker_limits_cpu: "400m"
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}"
      tags:
        - packit-worker
      when: workers_long_running > 0

    - name: Deploy packit-service-beat
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/packit-service-beat.yml.j2') }}"
      when: with_beat
      tags:
        - packit-service-beat

    - name: Deploy dashboard
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/dashboard.yml.j2') }}"
      when: with_dashboard
      tags:
        - dashboard

    - name: Create redis-commander secrets
      k8s:
        namespace: "{{ project }}"
        definition: "{{ lookup('template', '{{ project_dir }}/openshift/secret-redis-commander.yml.j2') }}"
        host: "{{ host }}"
        api_key: "{{ api_key }}"
        validate_certs: "{{ validate_certs }}"
        apply: true
      tags:
        - redis-commander
      notify:
        - Rollout redis-commander
      when: with_redis_commander

    - name: Deploy redis-commander
      vars:
        k8s_apply: true
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('file', '{{ project_dir }}/openshift/redis-commander.yml') }}"
      when: with_redis_commander
      tags:
        - redis-commander
      register: redis_commander

    - name: Create flower secret
      k8s:
        namespace: "{{ project }}"
        definition: "{{ lookup('template', '{{ project_dir }}/openshift/secret-flower.yml.j2') }}"
        host: "{{ host }}"
        api_key: "{{ api_key }}"
        validate_certs: "{{ validate_certs }}"
        apply: true
      tags:
        - flower
      notify:
        - Rollout flower
      when: with_flower

    - name: Deploy flower
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('file', '{{ project_dir }}/openshift/flower.yml') }}"
      when: with_flower
      tags:
        - flower
      register: flower

    - name: Deploy packit-service-fedmsg
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/packit-service-fedmsg.yml.j2') }}"
      tags:
        - fedmsg
      when: with_fedmsg

    - name: Deploy GitHub App Private Key
      k8s:
        namespace: "{{ project }}"
        resource_definition: "{{ lookup('template', '{{ project_dir }}/openshift/github-app-private-key.yml.j2')  }}"
        host: "{{ host }}"
        api_key: "{{ api_key }}"
        validate_certs: "{{ validate_certs }}"
      tags:
        - tokman
      notify:
        - Rollout tokman
      when: with_tokman

    - name: Deploy tokman
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/tokman.yml.j2') }}"
      tags:
        - tokman
      register: tokman
      when: with_tokman

    - name: Deploy aggregating pushgateway
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/pushgateway.yml.j2') }}"
      tags:
        - pushgateway
      when: with_pushgateway

    - name: Create htpasswd file and deploy it as a secret
      block:
        - name: Create htpasswd file
          htpasswd:
            path: "{{ flower_secret_htpasswd }}"
            name: "flower-boss"
            password: "{{ flower_basic_auth | regex_replace('flower-boss:', '') }}"
            mode: 0640
        - name: Deploy flower-htpasswd secret
          # Don't use tasks/k8s.yml here because the loop item is always evaluated
          k8s:
            namespace: "{{ project }}"
            resource_definition: "{{ lookup('template', '{{ project_dir }}/openshift/secret-flower-htpasswd.yml.j2') }}"
            host: "{{ host }}"
            api_key: "{{ api_key }}"
            validate_certs: "{{ validate_certs }}"
      tags:
        - flower
      when: with_flower

    - name: Deploy nginx to reverse proxy the pushgateway and flower
      include_tasks: tasks/k8s.yml
      loop:
        - "{{ lookup('template', '{{ project_dir }}/openshift/nginx.yml.j2') }}"
      tags:
        - pushgateway
      when: with_pushgateway and with_flower

    - name: Wait for worker-0 to be running
      vars:
        pod_name: packit-worker-0
      include_tasks: tasks/wait_for_pod.yml
      when: workers_all_tasks > 0

    - name: Wait for worker-short-running-0 to be running
      vars:
        pod_name: packit-worker-short-running-0
      include_tasks: tasks/wait_for_pod.yml
      when: workers_short_running > 0

    - name: Wait for worker-long-running-0 to be running
      vars:
        pod_name: packit-worker-long-running-0
      include_tasks: tasks/wait_for_pod.yml
      when: workers_long_running > 0

    - name: Wait for deploymentconfig rollouts to complete
      # timeout 10min to not wait indefinitely in case of a problem
      ansible.builtin.command: timeout 10m oc rollout status -w dc/{{ item }}
      register: oc_rollout_status
      changed_when: false
      failed_when: '"successfully rolled out" not in oc_rollout_status.stdout'
      loop: "{{ deploymentconfigs }}"

  handlers:
    - name: Rollout redis-commander
      ansible.builtin.command: oc rollout latest dc/redis-commander
      # Run this rollout only if the DeploymentConfig above didn't change,
      # and so it wasn't rolled out yet.
      when: not redis_commander.changed

    - name: Rollout tokman
      ansible.builtin.command: oc rollout latest dc/tokman
      # Run this rollout only if the DeploymentConfig above didn't change,
      # and so it wasn't rolled out yet.
      when: not tokman.changed

    - name: Rollout flower
      ansible.builtin.command: oc rollout latest dc/flower
      # Run this rollout only if the DeploymentConfig above didn't change,
      # and so it wasn't rolled out yet.
      when: not flower.changed