Skip to content
Permalink
Browse files
Merge pull request #29 from apache/switch-to-ami
Switch to using the built AMI instead of cloud-init
  • Loading branch information
potiuk committed May 17, 2021
2 parents d64610a + 4eb8f64 commit 82d9ba23db12f0bcd3e0c8b0a0ee5e6a9a133b19
Showing 1 changed file with 4 additions and 314 deletions.
@@ -1,329 +1,19 @@
#cloud-config
# 2021-03-20 14:00 CET
# 2021-05-17 09:21+01:00

mounts:
- [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"]
- [tmpfs, /var/lib/docker, tmpfs, "defaults,noatime,size=66%"]
- [tmpfs, /home/runner/actions-runner/_work, tmpfs, "defaults,noatime"]

users:
- default
- name: runner

packages:
- awscli
- build-essential
- docker.io
- git
- iptables-persistent
- jq
- parallel
- python3-dev
- python3-venv
- python3-wheel
- yarn
- vector

runcmd:
-
- bash
- -c
# https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/docker-compose.sh
- |
set -eu -o pipefail
URL=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(endswith("docker-compose-Linux-x86_64"))')
curl -L $URL -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
-
- bash
- -c
- |
set -eu -o pipefail
echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
# Set an env var (that is visible in runners) that will let us know we are on a self-hosted runner
echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment
set -a
. /etc/environment
set +a
echo "ASG_GROUP_NAME=$(aws ec2 describe-tags --filter Name=resource-id,Values=$(cloud-init query instance_id) Name=key,Values=aws:autoscaling:groupName \
| jq -r '@sh "\(.Tags[0].Value)"')" >> /etc/environment
- [systemctl, daemon-reload]
-
- bash
- -c
- |
set -eu -o pipefail
python3 -mvenv /opt/runner-supervisor
/opt/runner-supervisor/bin/pip install -U pip python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0'
-
- bash
- -c
- |
set -eu -o pipefail
usermod -G docker -a runner
mkdir -p ~runner/actions-runner
find ~runner -exec chown runner: {} +
cd ~runner/actions-runner
RUNNER_VERSION="$0"
curl -L "https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -zx
set -a
. /etc/environment
set +a
aws s3 cp s3://airflow-ci-assets/runner-supervisor.py /opt/runner-supervisor/bin/runner-supervisor
chmod 755 /opt/runner-supervisor/bin/runner-supervisor
- 2.277.1-airflow3
- [systemctl, enable, --now, iptables.service]
# Restart docker after applying the user firewall -- else some rules/chains might be list!
- [systemctl, restart, docker.service]
- [systemctl, enable, --now, vector.service]
- [systemctl, enable, --now, actions.runner.service]

write_files:
- path: /etc/systemd/system/actions.runner.service
content: |
[Unit]
Description=GitHub Actions Runner
After=network.target actions.runner-supervisor.service
Requires=actions.runner-supervisor.service
BindsTo=actions.runner-supervisor.service
[Service]
ExecStartPre=!/usr/local/sbin/runner-cleanup-workdir.sh
ExecStart=/home/runner/actions-runner/run.sh --once --startuptype service
ExecStop=/usr/local/bin/stop-runner-if-no-job.sh $MAINPID
EnvironmentFile=/etc/environment
Environment=GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT=300
User=runner
WorkingDirectory=/home/runner/actions-runner
KillMode=mixed
KillSignal=SIGTERM
TimeoutStopSec=5min
Restart=on-success
[Install]
WantedBy=multi-user.target
# Don't put this in ~runner, as these get written before the user is added, and this messes up creating the home dir
- path: /usr/local/sbin/runner-cleanup-workdir.sh
content: |
#!/bin/bash
set -eu -o pipefail
echo "Left-over containers:"
docker ps -a
docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv
echo "Log in to a paid docker user to get unlimited docker pulls"
aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \
jq .Parameter.Value -r | \
sudo -u runner docker login --username airflowcirunners --password-stdin
if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then
cd ~runner/actions-runner/_work/airflow/airflow
chown --changes -R runner: .
if [[ -e .git ]]; then
sudo -u runner bash -c "
git reset --hard && \
git submodule deinit --all -f && \
git submodule foreach git clean -fxd && \
git clean -fxd \
"
fi
fi
owner: root:root
permissions: '0775'
- path: /usr/local/bin/stop-runner-if-no-job.sh
content: |
#!/bin/bash
set -eu -o pipefail
MAINPID="${MAINPID:-${1:-}}"
if [[ -z "$MAINPID" ]]; then
echo "No MAINPID, assuming it already crashed!"
exit 0
fi
if pgrep --ns $MAINPID -a Runner.Worker > /dev/null; then
echo "Waiting for current job to finish"
while pgrep --ns $MAINPID -a Runner.Worker; do
# Job running -- just wait for it to exit
sleep 10
done
fi
# Request shutdown if it's still alive -- because we are in "stop" state it should not restart
if pkill --ns $MAINPID Runner.Listener; then
# Wait for it to shut down
echo "Waiting for main Runner.Listener process to stop"
while pgrep --ns $MAINPID -a Runner.Listener; do
sleep 5
done
fi
owner: root:root
permissions: '0775'
- path: /etc/sudoers.d/runner
owner: root:root
permissions: '0440'
content: |
runner ALL=(ALL) NOPASSWD:/usr/sbin/swapoff -a, /usr/bin/rm -f /swapfile, /usr/bin/apt clean
- path: /etc/iptables/rules.v4
content: |
# Generated by iptables-save v1.8.4 on Thu Jan 14 13:59:27 2021
*filter
:INPUT ACCEPT [833:75929]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [794:143141]
:DOCKER-USER - [0:0]
-A FORWARD -j DOCKER-USER
# Dis-allow any docker container to access the metadata service
-A DOCKER-USER -d 169.254.169.254/32 -j REJECT --reject-with icmp-port-unreachable
-A DOCKER-USER -j RETURN
COMMIT
- path: /usr/local/sbin/actions-runner-ec2-reporting
permissions: '0775'
content: |
#!/bin/bash
set -eu -o pipefail
if pgrep -c Runner.Worker >/dev/null; then
# Only report metric when we're doing something -- no point paying to submit zeros
aws cloudwatch put-metric-data --metric-name jobs-running --value "$(pgrep -c Runner.Worker)" --namespace github.actions
fi
- path: /etc/cron.d/cloudwatch-metrics-github-runners
content: |
*/1 * * * * nobody /usr/local/sbin/actions-runner-ec2-reporting
- path: /etc/systemd/system/actions.runner-supervisor.service
content: |
[Unit]
Description=Fetch credentials and supervise GitHub Actions Runner
After=network.target
Before=actions.runner.service
[Service]
Type=notify
ExecStart=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor
# We need to run as root to have the ability to open the netlink connector socket
User=root
WorkingDirectory=/home/runner/actions-runner
Restart=always
EnvironmentFile=/etc/environment
- path: /etc/vector/vector.toml
content: |
data_dir = "/var/lib/vector"
[api]
enabled = true
# Input data. Change me to a valid input source.
[sources.logs]
type = "journald"
include_units = ["actions.runner.service", "actions.runner-supervisor.service"]
[transforms.without_systemd_fields]
type = "remove_fields"
inputs = ["logs"]
fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP",
"_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID",
"_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT",
"__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY",
"source_type"]
[sources.runner-logs]
type = "file"
include = ["/home/runner/actions-runner/_diag/*.log"]
[sources.runner-logs.multiline]
start_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
mode = "halt_before"
condition_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
timeout_ms = 250
[transforms.grok-runner-logs]
type = "remap"
inputs=["runner-logs"]
source = '''
structured, err = parse_grok(.message, "(?m)\\[%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{NOTSPACE:logger}\\] %{GREEDYDATA:message}")
if err != null {
.err = err
} else {
. = merge(., structured)
}
'''
[transforms.filter-runner-logs]
type = "filter"
inputs = ['grok-runner-logs']
condition.type = "remap"
condition.source = '''
if .logger == "JobServerQueue" {
!match!(.message, r'Try to append \d+ batches web console lines for record')
} else if .logger == "HostContext" {
!starts_with!(.message, "Well known directory")
} else {
true
}
'''
[sources.job-logs]
type = "file"
include = ["/home/runner/actions-runner/_diag/pages/*.log"]
[transforms.grok-job-logs]
type = "remap"
inputs = ["job-logs"]
source = '''
structured, err = parse_grok(.message, "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}")
if err == null {
. = merge(., structured)
.message = strip_ansi_escape_codes!(.message)
.type = "job-output"
}
'''
# Output data
[sinks.cloudwatch]
inputs = ["without_systemd_fields", "filter-runner-logs", "grok-job-logs"]
type = "aws_cloudwatch_logs"
encoding = "json"
create_missing_group = false
create_missing_stream = true
group_name = "GitHubRunners"
stream_name = "{{ host }}"
region = "eu-central-1"
apt:
sources:
yarn:
source: "deb https://dl.yarnpkg.com/debian/ stable main"
keyid: "1646B01B86E50310"
timber:
source: "deb https://repositories.timber.io/public/vector/deb/ubuntu focal main"
key: |
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v2
mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ
jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d
2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy
Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL
Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz
629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj
a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE
EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor
xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X
muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE
RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA
gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm
0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g
3vq0NvX0k2CLi/W9mTiSdHXFChI15A==
=k36w
-----END PGP PUBLIC KEY BLOCK-----
echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/default/vector
systemctl enable --now vector
systemctl enable --now actions.runner

0 comments on commit 82d9ba2

Please sign in to comment.