cloud-init.yml

#cloud-config
# 2021-02-19 21:48

mounts:
  - [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"]
  - [tmpfs, /var/lib/docker, tmpfs, "defaults,noatime,size=66%"]
  - [tmpfs, /home/runner/actions-runner/_work, tmpfs, "defaults,noatime"]

users:
  - default
  - name: runner

packages:
  - awscli
  - build-essential
  - docker.io
  - git
  - iptables-persistent
  - jq
  - parallel
  - python3-dev
  - python3-venv
  - python3-wheel
  - yarn
  - vector

runcmd:
  -
    - bash
    - -c
    # https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/docker-compose.sh
    - |
      URL=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(endswith("docker-compose-Linux-x86_64"))')
      curl -L $URL -o /usr/local/bin/docker-compose
      chmod +x /usr/local/bin/docker-compose
  -
    - bash
    - -c
    - |
      set -exu -o pipefail
      echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
      # Set an env var (that is visible in runners) that will let us know we are on a self-hosted runner
      echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment
      set -a
      . /etc/environment
      set +a
      echo "ASG_GROUP_NAME=$(aws ec2 describe-tags --filter Name=resource-id,Values=$(cloud-init query instance_id) Name=key,Values=aws:autoscaling:groupName \
            | jq -r '@sh "\(.Tags[0].Value)"')" >> /etc/environment
  - [systemctl, daemon-reload]
  -
    - bash
    - -c
    - |
      python3 -mvenv /opt/runner-supervisor
      /opt/runner-supervisor/bin/pip install -U pip python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0'
  -
    - bash
    - -c
    - |
      set -exu -o pipefail

      usermod -G docker -a runner

      mkdir -p ~runner/actions-runner
      find ~runner -exec  chown runner: {} +
      cd ~runner/actions-runner

      RUNNER_VERSION="$0"

      curl -L "https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -zx

      set -a
      . /etc/environment
      set +a

      aws s3 cp s3://airflow-ci-assets/runner-supervisor.py /opt/runner-supervisor/bin/runner-supervisor
      chmod 755 /opt/runner-supervisor/bin/runner-supervisor

      # Log in to a paid docker user to get unlimited docker pulls
      aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \
        jq .Parameter.Value -r | \
        sudo -u runner docker login --username airflowcirunners --password-stdin
    - 2.277.1-airflow1
  - [systemctl, enable, --now, iptables.service]
  # Restart docker after applying the user firewall -- else some rules/chains might be list!
  - [systemctl, restart, docker.service]
  - [systemctl, enable, --now, vector.service]
  - [systemctl, enable, --now, actions.runner.service]
  -
    - bash
    - -c
    - |
      echo "Pre-loading commonly used docker images from S3"
      set -eux -o pipefail
      aws s3 cp s3://airflow-ci-assets/pre-baked-images.tar.gz - | docker load

write_files:
  - path: /etc/systemd/system/actions.runner.service
    content: |
      [Unit]
      Description=GitHub Actions Runner
      After=network.target actions.runner-supervisor.service
      Requires=actions.runner-supervisor.service
      BindsTo=actions.runner-supervisor.service

      [Service]
      ExecStartPre=!/usr/local/sbin/runner-cleanup-workdir.sh
      ExecStart=/home/runner/actions-runner/run.sh --once --startuptype service
      ExecStop=/usr/local/bin/stop-runner-if-no-job.sh $MAINPID
      EnvironmentFile=/etc/environment
      Environment=GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT=300
      User=runner
      WorkingDirectory=/home/runner/actions-runner
      KillMode=mixed
      KillSignal=SIGTERM
      TimeoutStopSec=5min
      Restart=on-success

      [Install]
      WantedBy=multi-user.target

  # Don't put this in ~runner, as these get written before the user is added, and this messes up creating the home dir
  - path: /usr/local/sbin/runner-cleanup-workdir.sh
    content: |
      #!/bin/bash
      echo "Left-over containers:"
      docker ps -a
      docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv

      if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then
        cd ~runner/actions-runner/_work/airflow/airflow

        chown --changes -R runner: .
        if [[ -e .git ]]; then
          sudo -u runner bash -c "
            git reset --hard && \
            git submodule deinit --all -f && \
            git submodule foreach git clean -fxd && \
            git clean -fxd \
          "
        fi
      fi
    owner: root:root
    permissions: '0775'
  - path: /usr/local/bin/stop-runner-if-no-job.sh
    content: |
      #!/bin/bash
      set -u

      MAINPID="${MAINPID:-${1:-}}"

      if [[ -z "$MAINPID" ]]; then
         echo "No MAINPID, assuming it already crashed!"
         exit 0
      fi

      if pgrep --ns $MAINPID -a Runner.Worker > /dev/null; then
        echo "Waiting for current job to finish"
        while pgrep --ns $MAINPID -a Runner.Worker; do
          # Job running -- just wait for it to exit
          sleep 10
        done
      fi

      # Request shutdown if it's still alive -- because we are in "stop" state it should not restart
      if pkill --ns $MAINPID Runner.Listener; then
        # Wait for it to shut down
        echo "Waiting for main Runner.Listener process to stop"
        while pgrep --ns $MAINPID -a Runner.Listener; do
          sleep 5
        done
      fi
    owner: root:root
    permissions: '0775'
  - path: /etc/sudoers.d/runner
    owner: root:root
    permissions: '0440'
    content: |
      runner ALL=(ALL) NOPASSWD:/usr/sbin/swapoff -a, /usr/bin/rm -f /swapfile, /usr/bin/apt clean
  - path: /etc/iptables/rules.v4
    content: |
      # Generated by iptables-save v1.8.4 on Thu Jan 14 13:59:27 2021
      *filter
      :INPUT ACCEPT [833:75929]
      :FORWARD DROP [0:0]
      :OUTPUT ACCEPT [794:143141]
      :DOCKER-USER - [0:0]
      -A FORWARD -j DOCKER-USER
      # Dis-allow any docker container to access the metadata service
      -A DOCKER-USER -d 169.254.169.254/32 -j REJECT --reject-with icmp-port-unreachable
      -A DOCKER-USER -j RETURN
      COMMIT

  - path: /usr/local/sbin/actions-runner-ec2-reporting
    permissions: '0775'
    content: |
      #!/bin/bash

      if pgrep -c Runner.Worker >/dev/null; then
          # Only report metric when we're doing something -- no point paying to submit zeros
          aws cloudwatch put-metric-data --metric-name jobs-running --value "$(pgrep -c Runner.Worker)" --namespace github.actions
      fi
  - path: /etc/cron.d/cloudwatch-metrics-github-runners
    content: |
      */1 * * * * nobody /usr/local/sbin/actions-runner-ec2-reporting

  - path: /etc/systemd/system/actions.runner-supervisor.service
    content: |
      [Unit]
      Description=Fetch credentials and supervise GitHub Actions Runner
      After=network.target
      Before=actions.runner.service

      [Service]
      Type=notify
      ExecStart=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor
      # We need to run as root to have the ability to open the netlink connector socket
      User=root
      WorkingDirectory=/home/runner/actions-runner
      Restart=always
      EnvironmentFile=/etc/environment
  - path: /etc/vector/vector.toml
    content: |
      data_dir = "/var/lib/vector"

      [api]
        enabled = true

      # Input data. Change me to a valid input source.
      [sources.logs]
        type = "journald"
        include_units = ["actions.runner.service", "actions.runner-supervisor.service"]

      [transforms.without_systemd_fields]
        type = "remove_fields"
        inputs = ["logs"]
        fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP",
          "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID",
          "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT",
          "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY",
          "source_type"]

      [sources.runner-logs]
        type = "file"
        include = ["/home/runner/actions-runner/_diag/*.log"]

          [sources.runner-logs.multiline]
            start_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
            mode = "halt_before"
            condition_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
            timeout_ms = 250

      [transforms.grok-runner-logs]
        type = "remap"
        inputs=["runner-logs"]
        source = '''
          structured, err = parse_grok(.message, "(?m)\\[%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{NOTSPACE:logger}\\] %{GREEDYDATA:message}")

          if err != null {
            .err = err
          } else {
            . = merge(., structured)
          }
        '''
      [transforms.filter-runner-logs]
        type = "filter"
        inputs = ['grok-runner-logs']
        condition.type = "remap"
        condition.source = '''
          if .logger == "JobServerQueue" {
            !match!(.message, r'Try to append \d+ batches web console lines for record')
          } else if .logger == "HostContext" {
            !starts_with!(.message, "Well known directory")
          } else {
            true
          }
        '''

      [sources.job-logs]
        type = "file"
        include = ["/home/runner/actions-runner/_diag/pages/*.log"]

      [transforms.grok-job-logs]
        type = "remap"
        inputs = ["job-logs"]
        source = '''
          structured, err = parse_grok(.message, "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}")

          if err == null {
            . = merge(., structured)
            .type = "job-output"
          }
        '''

      # Output data
      [sinks.cloudwatch]
      inputs   = ["without_systemd_fields", "filter-runner-logs", "grok-job-logs"]
      type     = "aws_cloudwatch_logs"
      encoding = "json"
      create_missing_group = false
      create_missing_stream = true
      group_name = "GitHubRunners"
      stream_name = "{{ host }}"
      region = "eu-central-1"
apt:
  sources:
    yarn:
      source: "deb https://dl.yarnpkg.com/debian/ stable main"
      keyid: "1646B01B86E50310"
    timber:
      source: "deb https://repositories.timber.io/public/vector/deb/ubuntu focal main"
      key: |
        -----BEGIN PGP PUBLIC KEY BLOCK-----
        Version: GnuPG v2

        mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ
        jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d
        2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy
        Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL
        Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz
        629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj
        a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE
        EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor
        xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X
        muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE
        RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA
        gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm
        0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g
        3vq0NvX0k2CLi/W9mTiSdHXFChI15A==
        =k36w
        -----END PGP PUBLIC KEY BLOCK-----