Merge pull request #29 from apache/switch-to-ami

Switch to using the built AMI instead of cloud-init

cloud-init.yml

@@ -1,329 +1,19 @@
 #cloud-config
-# 2021-03-20 14:00 CET
+# 2021-05-17 09:21+01:00
 
 mounts:
   - [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"]
   - [tmpfs, /var/lib/docker, tmpfs, "defaults,noatime,size=66%"]
   - [tmpfs, /home/runner/actions-runner/_work, tmpfs, "defaults,noatime"]
 
-users:
-  - default
-  - name: runner
-
-packages:
-  - awscli
-  - build-essential
-  - docker.io
-  - git
-  - iptables-persistent
-  - jq
-  - parallel
-  - python3-dev
-  - python3-venv
-  - python3-wheel
-  - yarn
-  - vector
 
 runcmd:
-  -
-    - bash
-    - -c
-    # https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/docker-compose.sh
-    - |
-      set -eu -o pipefail
-      URL=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(endswith("docker-compose-Linux-x86_64"))')
-      curl -L $URL -o /usr/local/bin/docker-compose
-      chmod +x /usr/local/bin/docker-compose
   -
     - bash
     - -c
     - |
       set -eu -o pipefail
       echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
-      # Set an env var (that is visible in runners) that will let us know we are on a self-hosted runner
-      echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment
-      set -a
-      . /etc/environment
-      set +a
-      echo "ASG_GROUP_NAME=$(aws ec2 describe-tags --filter Name=resource-id,Values=$(cloud-init query instance_id) Name=key,Values=aws:autoscaling:groupName \
-            | jq -r '@sh "\(.Tags[0].Value)"')" >> /etc/environment
-  - [systemctl, daemon-reload]
-  -
-    - bash
-    - -c
-    - |
-      set -eu -o pipefail
-      python3 -mvenv /opt/runner-supervisor
-      /opt/runner-supervisor/bin/pip install -U pip python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0'
-  -
-    - bash
-    - -c
-    - |
-      set -eu -o pipefail
-
-      usermod -G docker -a runner
-
-      mkdir -p ~runner/actions-runner
-      find ~runner -exec  chown runner: {} +
-      cd ~runner/actions-runner
-
-      RUNNER_VERSION="$0"
-
-      curl -L "https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -zx
-
-      set -a
-      . /etc/environment
-      set +a
-
-      aws s3 cp s3://airflow-ci-assets/runner-supervisor.py /opt/runner-supervisor/bin/runner-supervisor
-      chmod 755 /opt/runner-supervisor/bin/runner-supervisor
-
-    - 2.277.1-airflow3
-  - [systemctl, enable, --now, iptables.service]
-  # Restart docker after applying the user firewall -- else some rules/chains might be list!
-  - [systemctl, restart, docker.service]
-  - [systemctl, enable, --now, vector.service]
-  - [systemctl, enable, --now, actions.runner.service]
-
-write_files:
-  - path: /etc/systemd/system/actions.runner.service
-    content: |
-      [Unit]
-      Description=GitHub Actions Runner
-      After=network.target actions.runner-supervisor.service
-      Requires=actions.runner-supervisor.service
-      BindsTo=actions.runner-supervisor.service
-
-      [Service]
-      ExecStartPre=!/usr/local/sbin/runner-cleanup-workdir.sh
-      ExecStart=/home/runner/actions-runner/run.sh --once --startuptype service
-      ExecStop=/usr/local/bin/stop-runner-if-no-job.sh $MAINPID
-      EnvironmentFile=/etc/environment
-      Environment=GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT=300
-      User=runner
-      WorkingDirectory=/home/runner/actions-runner
-      KillMode=mixed
-      KillSignal=SIGTERM
-      TimeoutStopSec=5min
-      Restart=on-success
-
-      [Install]
-      WantedBy=multi-user.target
-
-  # Don't put this in ~runner, as these get written before the user is added, and this messes up creating the home dir
-  - path: /usr/local/sbin/runner-cleanup-workdir.sh
-    content: |
-      #!/bin/bash
-      set -eu -o pipefail
-      echo "Left-over containers:"
-      docker ps -a
-      docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv
-
-      echo "Log in to a paid docker user to get unlimited docker pulls"
-      aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \
-        jq .Parameter.Value -r | \
-        sudo -u runner docker login --username airflowcirunners --password-stdin
-
-      if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then
-        cd ~runner/actions-runner/_work/airflow/airflow
-
-        chown --changes -R runner: .
-        if [[ -e .git ]]; then
-          sudo -u runner bash -c "
-            git reset --hard && \
-            git submodule deinit --all -f && \
-            git submodule foreach git clean -fxd && \
-            git clean -fxd \
-          "
-        fi
-      fi
-    owner: root:root
-    permissions: '0775'
-  - path: /usr/local/bin/stop-runner-if-no-job.sh
-    content: |
-      #!/bin/bash
-      set -eu -o pipefail
-
-      MAINPID="${MAINPID:-${1:-}}"
-
-      if [[ -z "$MAINPID" ]]; then
-         echo "No MAINPID, assuming it already crashed!"
-         exit 0
-      fi
-
-      if pgrep --ns $MAINPID -a Runner.Worker > /dev/null; then
-        echo "Waiting for current job to finish"
-        while pgrep --ns $MAINPID -a Runner.Worker; do
-          # Job running -- just wait for it to exit
-          sleep 10
-        done
-      fi
-
-      # Request shutdown if it's still alive -- because we are in "stop" state it should not restart
-      if pkill --ns $MAINPID Runner.Listener; then
-        # Wait for it to shut down
-        echo "Waiting for main Runner.Listener process to stop"
-        while pgrep --ns $MAINPID -a Runner.Listener; do
-          sleep 5
-        done
-      fi
-    owner: root:root
-    permissions: '0775'
-  - path: /etc/sudoers.d/runner
-    owner: root:root
-    permissions: '0440'
-    content: |
-      runner ALL=(ALL) NOPASSWD:/usr/sbin/swapoff -a, /usr/bin/rm -f /swapfile, /usr/bin/apt clean
-  - path: /etc/iptables/rules.v4
-    content: |
-      # Generated by iptables-save v1.8.4 on Thu Jan 14 13:59:27 2021
-      *filter
-      :INPUT ACCEPT [833:75929]
-      :FORWARD DROP [0:0]
-      :OUTPUT ACCEPT [794:143141]
-      :DOCKER-USER - [0:0]
-      -A FORWARD -j DOCKER-USER
-      # Dis-allow any docker container to access the metadata service
-      -A DOCKER-USER -d 169.254.169.254/32 -j REJECT --reject-with icmp-port-unreachable
-      -A DOCKER-USER -j RETURN
-      COMMIT
-
-  - path: /usr/local/sbin/actions-runner-ec2-reporting
-    permissions: '0775'
-    content: |
-      #!/bin/bash
-      set -eu -o pipefail
-      if pgrep -c Runner.Worker >/dev/null; then
-          # Only report metric when we're doing something -- no point paying to submit zeros
-          aws cloudwatch put-metric-data --metric-name jobs-running --value "$(pgrep -c Runner.Worker)" --namespace github.actions
-      fi
-  - path: /etc/cron.d/cloudwatch-metrics-github-runners
-    content: |
-      */1 * * * * nobody /usr/local/sbin/actions-runner-ec2-reporting
-
-  - path: /etc/systemd/system/actions.runner-supervisor.service
-    content: |
-      [Unit]
-      Description=Fetch credentials and supervise GitHub Actions Runner
-      After=network.target
-      Before=actions.runner.service
-
-      [Service]
-      Type=notify
-      ExecStart=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor
-      # We need to run as root to have the ability to open the netlink connector socket
-      User=root
-      WorkingDirectory=/home/runner/actions-runner
-      Restart=always
-      EnvironmentFile=/etc/environment
-  - path: /etc/vector/vector.toml
-    content: |
-      data_dir = "/var/lib/vector"
-
-      [api]
-        enabled = true
-
-      # Input data. Change me to a valid input source.
-      [sources.logs]
-        type = "journald"
-        include_units = ["actions.runner.service", "actions.runner-supervisor.service"]
-
-      [transforms.without_systemd_fields]
-        type = "remove_fields"
-        inputs = ["logs"]
-        fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP",
-          "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID",
-          "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT",
-          "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY",
-          "source_type"]
-
-      [sources.runner-logs]
-        type = "file"
-        include = ["/home/runner/actions-runner/_diag/*.log"]
-
-          [sources.runner-logs.multiline]
-            start_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
-            mode = "halt_before"
-            condition_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
-            timeout_ms = 250
-
-      [transforms.grok-runner-logs]
-        type = "remap"
-        inputs=["runner-logs"]
-        source = '''
-          structured, err = parse_grok(.message, "(?m)\\[%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{NOTSPACE:logger}\\] %{GREEDYDATA:message}")
-
-          if err != null {
-            .err = err
-          } else {
-            . = merge(., structured)
-          }
-        '''
-      [transforms.filter-runner-logs]
-        type = "filter"
-        inputs = ['grok-runner-logs']
-        condition.type = "remap"
-        condition.source = '''
-          if .logger == "JobServerQueue" {
-            !match!(.message, r'Try to append \d+ batches web console lines for record')
-          } else if .logger == "HostContext" {
-            !starts_with!(.message, "Well known directory")
-          } else {
-            true
-          }
-        '''
-
-      [sources.job-logs]
-        type = "file"
-        include = ["/home/runner/actions-runner/_diag/pages/*.log"]
-
-      [transforms.grok-job-logs]
-        type = "remap"
-        inputs = ["job-logs"]
-        source = '''
-          structured, err = parse_grok(.message, "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}")
-
-          if err == null {
-            . = merge(., structured)
-            .message = strip_ansi_escape_codes!(.message)
-            .type = "job-output"
-          }
-        '''
-
-      # Output data
-      [sinks.cloudwatch]
-      inputs   = ["without_systemd_fields", "filter-runner-logs", "grok-job-logs"]
-      type     = "aws_cloudwatch_logs"
-      encoding = "json"
-      create_missing_group = false
-      create_missing_stream = true
-      group_name = "GitHubRunners"
-      stream_name = "{{ host }}"
-      region = "eu-central-1"
-apt:
-  sources:
-    yarn:
-      source: "deb https://dl.yarnpkg.com/debian/ stable main"
-      keyid: "1646B01B86E50310"
-    timber:
-      source: "deb https://repositories.timber.io/public/vector/deb/ubuntu focal main"
-      key: |
-        -----BEGIN PGP PUBLIC KEY BLOCK-----
-        Version: GnuPG v2
-
-        mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ
-        jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d
-        2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy
-        Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL
-        Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz
-        629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj
-        a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE
-        EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor
-        xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X
-        muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE
-        RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA
-        gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm
-        0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g
-        3vq0NvX0k2CLi/W9mTiSdHXFChI15A==
-        =k36w
-        -----END PGP PUBLIC KEY BLOCK-----
+      echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/default/vector
+      systemctl enable --now vector
+      systemctl enable --now actions.runner