Skip to content
Permalink
master
Switch branches/tags
Go to file
 
 
Cannot retrieve contributors at this time
#cloud-config
package_update: true
package_upgrade: true
packages: ['prometheus', 'prometheus-node-exporter', 'awscli', 'inotify-tools', 'nginx', 'jq']
write_files:
- owner: root:root
path: /etc/default/prometheus
permissions: 0444
content: 'ARGS="--storage.tsdb.path=\"/mnt/\" --web.external-url=${prom_external_url} --storage.tsdb.retention=60d --query.timeout=30s"'
- owner: root:root
path: /etc/cron.d/config_pull
permissions: 0755
content: |
* * * * * root flock -w 30 /run/lock/prometheus-config-updates aws s3 sync s3://${config_bucket}/prometheus/ /etc/prometheus/ --region=${region}
@reboot root /root/watch_prometheus_dir
- owner: root:root
path: /etc/cron.d/ireland_targets_pull
permissions: 0755
content: |
# if targets bucket exists then sync it, otherwise this cron runs but has no effect
* * * * * root [ "${ireland_targets_bucket}" != "" ] && aws s3 sync s3://${ireland_targets_bucket}/active/ /etc/prometheus/ireland-targets --region=${region} --delete
- owner: root:root
path: /etc/cron.d/london_targets_pull
permissions: 0755
content: |
# if targets bucket exists then sync it, otherwise this cron runs but has no effect
* * * * * root [ "${london_targets_bucket}" != "" ] && aws s3 sync s3://${london_targets_bucket}/active/ /etc/prometheus/london-targets --region=${region} --delete
- owner: root:root
path: /etc/cron.d/alerts_pull
permissions: 0755
content: |
# if alerts bucket exists then sync it, otherwise this cron runs but has no effect
* * * * * root [ "${alerts_bucket}" != "" ] && aws s3 sync s3://${alerts_bucket}/prometheus/alerts/ /etc/prometheus/alerts --region=${region} --delete
- content: |
echo 'Configuring prometheus EBS'
vol=""
while [ -z "$vol" ]; do
# adapted from
# https://medium.com/@moonape1226/mount-aws-ebs-on-ec2-automatically-with-cloud-init-e5e837e5438a
# [Last accessed on 2020-04-02]
vol=$(lsblk | grep -e disk | awk '{sub("G","",$4)} {if ($4+0 == ${data_volume_size}) print $1}')
echo "still waiting for data volume ; sleeping 5"
sleep 5
done
echo "found volume /dev/$vol"
if [ -z "$(lsblk | grep "$vol" | awk '{print $7}')" ] ; then
if [ -z "$(blkid /dev/$vol | grep ext4)" ] ; then
echo "volume /dev/$vol is not formatted ; formatting"
mkfs -F -t ext4 -L 'prometheus_disk' "/dev/$vol"
else
echo "volume /dev/$vol is already formatted"
fi
echo "volume /dev/$vol is not mounted ; mounting"
mount "/dev/$vol" /mnt
UUID=$(blkid /dev/$vol -s UUID -o value)
if [ -z "$(grep $UUID /etc/fstab)" ] ; then
echo "writing fstab entry"
echo "UUID=$UUID /mnt ext4 defaults,nofail 0 2" >> /etc/fstab
fi
fi
echo "ensuring fs block size matches volume block size"
resize2fs "/dev/$vol"
path: /root/manage_data_volume.sh
permissions: 0755
- content: |
#!/bin/bash
STATUS_JSON='/srv/prometheus-last-config.json'
attempt_reload() {
(
# take out lock to ensure updater doesn't switch the config between the time we
# calculate NEW_HASH and prometheus reads it
flock 321
# why md5? because it should be the same as the s3 etag and so easy to check
export NEW_HASH=$(md5sum /etc/prometheus/prometheus.yml | cut -d ' ' -f 1)
if systemctl reload prometheus ; then
jq -n '{last_successful_config: env.NEW_HASH, last_reload_successful: true}' > $STATUS_JSON
else
touch $STATUS_JSON
jq '{last_successful_config: .last_successful_config, last_reload_successful: false, failed_config: env.NEW_HASH}' $STATUS_JSON > $STATUS_JSON
fi
) 321>/run/lock/prometheus-config-updates
}
systemctl start prometheus # ensure prometheus is started before initial attempt_reload
attempt_reload
inotifywait -e modify,create,delete,move -m /etc/prometheus |
while read -r directory events; do
attempt_reload
done
path: /root/watch_prometheus_dir
permissions: 0755
- content: |
#!/bin/bash
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.4.2-amd64.deb && sudo dpkg -i filebeat-6.4.2-amd64.deb
aws s3 sync s3://${config_bucket}/filebeat/ /etc/filebeat/ --region=${region}
update-rc.d filebeat defaults
update-rc.d filebeat enable 5
path: /root/setup_filebeat.sh
permissions: 0755
- content: |
server {
listen 8080;
location / {
set $cleaned_header $arg_cf_app_instance;
if ($arg_cf_app_instance ~* "^(.*)%3A(.*)$") {
set $cleaned_header $1:$2;
}
proxy_http_version 1.1;
proxy_pass https://$host$uri;
proxy_ssl_server_name on;
proxy_set_header Connection "";
proxy_set_header X-CF-APP-INSTANCE $cleaned_header;
proxy_set_header XX-CF-APP-INSTANCE $cleaned_header;
proxy_set_header Authorization "Bearer $arg_cf_app_guid";
}
location /health {
return 200 "Static health check";
}
resolver 10.0.0.2 valid=10s;
}
path: /etc/nginx/sites-enabled/paas-proxy
permissions: 0644
- content: |
${prometheus_htpasswd}
path: /etc/nginx/conf.d/.htpasswd
owner: www-data:www-data
permissions: 0600
# the package-provided default server conflicts with auth-proxy
# below and causes package installation to fail because of a
# duplicate default_server on port 80. So we wipe the default
# server (and then remove it in runcmd at the bottom)
- content: ""
path: /etc/nginx/sites-enabled/default
- content: |
server {
listen 80 default_server;
location /health {
# This location is not protected by basic auth because of
# https://stackoverflow.com/questions/40447376/auth-basic-within-location-block-doesnt-work-when-return-is-specified
return 200 "Static health check";
}
location = /last-config {
default_type application/json;
alias /srv/prometheus-last-config.json;
}
location / {
proxy_pass http://localhost:9090;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
satisfy any;
auth_basic "Prometheus";
auth_basic_user_file /etc/nginx/conf.d/.htpasswd;
real_ip_header X-Forwarded-For;
set_real_ip_from 10.0.0.0/8;
set_real_ip_from 127.0.0.1/32;
${allowed_cidrs}
deny all;
}
path: /etc/nginx/sites-enabled/auth-proxy
runcmd:
- rm /etc/nginx/sites-enabled/default
- "if [ -n '${logstash_host}' ]; then /root/setup_filebeat.sh; fi"
- [bash, -c, "/root/manage_data_volume.sh"]
- [bash, -c, "chown -R prometheus /mnt/"]
- [bash, -c, "echo \"node_creation_time `date +%s`\" > /var/lib/prometheus/node-exporter/node-creation-time.prom"]
- [bash, -c, "rm /etc/resolv.conf && sed -e 's/ trust-ad//' < /run/systemd/resolve/stub-resolv.conf > /etc/resolv.conf"]
- [reboot]