Permalink
Browse files

chore(cm:*): checkpoint a bunch of disparate changes from various

branches
  • Loading branch information...
Lee Hambley
Lee Hambley committed Dec 7, 2017
1 parent f7dcfec commit 0a36b40c85d308d9ddeeeb2a05fc440778a06aec
Showing with 887 additions and 792 deletions.
  1. +2 −0 config-management/inventories/.gitignore
  2. +11 −7 config-management/provision.yml
  3. +12 −9 config-management/roles/harrow.backend/defaults/main.yml
  4. +16 −0 config-management/roles/harrow.backend/tasks/harrow-backup-executable.yml
  5. +0 −7 config-management/roles/harrow.backend/tasks/lxd-hosts.yml
  6. +1 −1 config-management/roles/harrow.backend/tasks/main.yml
  7. +12 −6 config-management/roles/harrow.backend/templates/etc/harrow/env.j2
  8. +10 −3 config-management/roles/harrow.backend/templates/etc/harrow/env.sh.j2
  9. +57 −0 config-management/roles/harrow.backend/templates/usr/local/bin/harrow-backup.j2
  10. +65 −0 config-management/roles/harrow.backend/templates/usr/local/bin/harrow-restore.j2
  11. +2 −0 config-management/roles/harrow.base/files/authorized_keys
  12. +51 −0 config-management/roles/harrow.base/tasks/centralized-logging.yml
  13. +9 −0 config-management/roles/harrow.base/tasks/email-routing.yml
  14. +1 −0 config-management/roles/harrow.base/tasks/main.yml
  15. +9 −3 config-management/roles/harrow.base/tasks/no-outgoing-cron-mail.yml
  16. +0 −19 config-management/roles/harrow.graylog/files/var/lib/harrow/graylog/docker-compose.yml
  17. 0 config-management/roles/harrow.graylog/tasks/main.yml
  18. +4 −0 config-management/roles/harrow.http-server/tasks/main.yml
  19. +25 −0 config-management/roles/harrow.metrics-host/files/elasticsearch.yml
  20. +29 −4 config-management/roles/harrow.metrics-host/tasks/main.yml
  21. +18 −0 config-management/roles/harrow.metrics-host/templates/Caddyfile.j2
  22. +14 −2 config-management/roles/harrow.metrics-host/templates/docker-compose.yml
  23. +9 −0 config-management/roles/harrow.postgresql-server/tasks/main.yml
  24. +11 −36 config-management/roles/harrow.postgresql-server/tasks/pgbouncer.yml
  25. +0 −229 config-management/roles/harrow.postgresql-server/templates/etc/pgbouncer/pgbouncer.ini.j2
  26. +0 −3 config-management/roles/harrow.postgresql-server/templates/etc/pgbouncer/userlist.txt.j2
  27. +21 −0 config-management/roles/harrow.postgresql-server/templates/usr/local/bin/harrow-backup-postresql.j2
  28. +3 −0 config-management/roles/harrow.rabbitmq-master/tasks/main.yml
  29. +9 −0 config-management/roles/harrow.redis-master/tasks/main.yml
  30. +9 −0 config-management/roles/harrow.redis-master/templates/usr/local/bin/harrow-backup-redis.j2
  31. +22 −22 config-management/stow/intern/group_vars/all/shared.yml
  32. +313 −305 config-management/stow/intern/group_vars/all/vault.yml
  33. +8 −6 config-management/stow/intern/group_vars/base.yml
  34. +12 −12 config-management/stow/intern/group_vars/http.yml
  35. +21 −17 config-management/stow/intern/group_vars/services.yml
  36. +8 −8 config-management/stow/intern/host_vars/alaska.yml
  37. +9 −9 config-management/stow/intern/host_vars/albert.yml
  38. +9 −9 config-management/stow/intern/host_vars/albino.yml
  39. +37 −37 config-management/stow/intern/host_vars/alcohol.yml
  40. +9 −9 config-management/stow/intern/inventories/enterprise
  41. +29 −29 config-management/stow/intern/inventories/production
@@ -0,0 +1,2 @@
enterprise
production
@@ -35,25 +35,29 @@
roles:
- { role: dj-wasabi.telegraf }
vars:
telegraf_agent_hostname: "{{ ansible_nodename }}"
telegraf_agent_output:
- type: influxdb
config:
- urls = ["http://{{ vault.influxdb.basicauth.username }}:{{ vault.influxdb.basicauth.password }}@{{ vault.influxdb.host }}:{{ vault.influxdb.port }}"]
- urls = ["https://{{ vault.influxdb.basicauth.username }}:{{ vault.influxdb.basicauth.password }}@{{ vault.influxdb.host }}"]
- database = "telegraf"
tagpass:
- diskmetrics = ["true"]
telegraf_plugins_default:
telegraf_plugins_extra:
- plugin: redis
# - plugin: rabbitmq
# config:
# - username = "{{ shared.rabbitmq.production.username }}"
# - password = "{{ vault.rabbitmq.production.password }}"
# TODO: ensure we can loop here, or something - kinda annoying that I can't loop over
# variables in yaml itself
# - plugin: postgresql
# - address = "postgres://{{ pg.username }}:{{ pg.password }}@{{ pg.host }}:{{ pg.port }}/{{ pg.name }}?sslmode=disable"
- plugin: rabbitmq
config:
- username = "{{ shared.rabbitmq.production.username }}"
- password = "{{ vault.rabbitmq.production.password }}"
tags:
- metrics
- hosts: all
roles:
- harrow.base
- franklinkim.ufw
tags:
- base
@@ -14,6 +14,7 @@ harrow:
services:
notify_on_failure: false
victorops:
enabled: "{{ vault.victorops.enabled }}"
api_id: "{{ vault.victorops.api_id }}"
@@ -47,8 +48,8 @@ harrow:
enabled: true
authentication:
enabled: true
app_id: {{ vault.github.oauth.app_id }}
app_secret: {{ vault.github.oauth.app_secret }}
app_id: "{{ vault.github.oauth.app_id }}"
app_secret: "{{ vault.github.oauth.app_secret }}"
redirect_uri_pattern: 'https://www.${domain}/#/a/github/callback/%s'
features:
@@ -60,6 +61,8 @@ harrow:
enabled: true
public_projects:
enabled: true
billing:
enabled: true
rabbitmq:
virtual_hosts:
@@ -85,26 +88,26 @@ harrow:
postgresql:
databases:
- environment: test
host: localhost
name: harrow_test
password: test
port: 5432
sslmode: 'disable'
host: /var/run/postgresql
username: test
- environment: development
host: localhost
name: "{{ shared.postgresql.development.name }}"
username: "{{ shared.postgresql.development.username }}"
password: "{{ vault.postgresql.development.password }}"
port: 6432
host: /var/run/postgresql
port: 5432
sslmode: 'disable'
username: "{{ shared.postgresql.development.username }}"
- environment: production
host: localhost
name: "{{ shared.postgresql.production.name }}"
username: "{{ shared.postgresql.production.username }}"
password: "{{ vault.postgresql.production.password }}"
port: 6432
host: /var/run/postgresql
port: 5432
sslmode: 'disable'
username: "{{ shared.postgresql.production.username }}"
environment: production
@@ -0,0 +1,16 @@
---
- name: harrow-backup/restore executable
template:
src: "usr/local/bin/{{ item }}.j2"
dest: "/usr/local/bin/{{ item }}"
mode: 0766
with_items:
- harrow-backup
- harrow-restore
- name: backup operation-logs
cron:
name: backup op-logs
special_time: daily
job: "/usr/local/bin/harrow-backup op-logs"

This file was deleted.

Oops, something went wrong.
@@ -4,7 +4,7 @@
when: harrow.backend.version
- include: config.yml
- include: harrow-backup-executable.yml
- include: legacy-notifier-wrappers.yml
- include: lxd-hosts.yml
- include: runner-service.yml
- include: services.yml
@@ -22,20 +22,26 @@ HAR_OPERATOR_SSH_KEY_FILENAME={{ harrow.operator.filename }}
HAR_FILESYSTEM_OP_LOG_DIR={{ harrow.filesystem.op_log_dir }}
HAR_FILESYSTEM_GIT_TMP_DIR={{ harrow.filesystem.git_tmp_dir }}
# Generate an exampel with `openssl rand -hex 50'
# Generate an example with `openssl rand -hex 50'
HAR_HTTP_USER_HMAC_SECRET={{ vault.http.user_hmac_secret }}
HAR_LIMIT_STORE_CACHE_DIR=/tmp
HAR_MAIL_FROM_ADDRESS=notifications@harrow.io
{% if harrow.github.oauth.app_id %}
{% if harrow.github.oauth.app_id -%}
HAR_OAUTH_GITHUB_CLIENT_ID={{ harrow.github.oauth.app_id }}
{% endif %}
{% if harrow.github.oauth.app_secret %}
{% endif -%}
{% if harrow.github.oauth.app_secret -%}
HAR_OAUTH_GITHUB_CLIENT_SECRET={{ harrow.github.oauth.app_secret }}
{% endif %}
{% endif -%}
HAR_OAUTH_GITHUB_PROVIDER_URL=https://github.com/login/oauth/authorize
HAR_OAUTH_GITHUB_REDIRECT_URI={{ harrow.github.oauth.redirect_uri_pattern }}
HAR_OAUTH_GITHUB_SCOPE=user,write:repo_hook,write:public_key,repo
{% if harrow.features.billing.enabled -%}
HAR_BRAINTREE_ENVIRONMENT={{ vault.braintree.environment }}
HAR_BRAINTREE_MERCHANT_ID={{ vault.braintree.merchant_id }}
HAR_BRAINTREE_PUBLIC_KEY={{ vault.braintree.public_key }}
HAR_BRAINTREE_PRIVATE_KEY={{ vault.braintree.private_key }}
{% endif -%}
@@ -22,7 +22,7 @@ export HAR_OPERATOR_SSH_KEY_FILENAME={{ harrow.operator.filename }}
export HAR_FILESYSTEM_OP_LOG_DIR={{ harrow.filesystem.op_log_dir }}
export HAR_FILESYSTEM_GIT_TMP_DIR={{ harrow.filesystem.git_tmp_dir }}
# Generate an exampel with `openssl rand -hex 50'
# Generate an example with `openssl rand -hex 50'
export HAR_HTTP_USER_HMAC_SECRET={{ vault.http.user_hmac_secret }}
export HAR_LIMIT_STORE_CACHE_DIR=/tmp
@@ -32,10 +32,17 @@ export HAR_MAIL_FROM_ADDRESS=notifications@harrow.io
export HAR_OAUTH_GITHUB_CLIENT_ID={{ harrow.github.oauth.app_id }}
{% endif %}
{% if harrow.github.oauth.app_secret %}
{% if harrow.github.oauth.app_secret -%}
export HAR_OAUTH_GITHUB_CLIENT_SECRET={{ harrow.github.oauth.app_secret }}
{% endif %}
{% endif -%}
export HAR_OAUTH_GITHUB_PROVIDER_URL=https://github.com/login/oauth/authorize
export HAR_OAUTH_GITHUB_REDIRECT_URI={{ harrow.github.oauth.redirect_uri_pattern }}
export HAR_OAUTH_GITHUB_SCOPE=user,write:repo_hook,write:public_key,repo
{% if harrow.features.billing.enabled -%}
export HAR_BRAINTREE_ENVIRONMENT={{ vault.braintree.environment }}
export HAR_BRAINTREE_MERCHANT_ID={{ vault.braintree.merchant_id }}
export HAR_BRAINTREE_PUBLIC_KEY={{ vault.braintree.public_key }}
export HAR_BRAINTREE_PRIVATE_KEY={{ vault.braintree.private_key }}
{% endif -%}
@@ -0,0 +1,57 @@
#!/bin/bash -e
# vim: ai ts=2 sw=2 et sts=2 ft=sh
# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=sh
export AWS_ACCESS_KEY_ID={{ vault.backup.aws_access_key_id }}
export AWS_SECRET_ACCESS_KEY={{ vault.backup.aws_secret_access_key }}
export AWS_DEFAULT_REGION={{ vault.backup.aws_region }}
function backup_op_logs {
/usr/bin/aws s3 sync \
--sse=aws:kms \
--storage-class=STANDARD_IA \
"/var/lib/harrow/op-logs/" \
"s3://{{ vault.backup.aws_s3_bucket_name }}/operation-logs-hetzner/"
}
function backup_redis {
dirname=redis-hetzner/$(date --iso-8601=minutes)
/usr/bin/aws s3 cp \
--sse=aws:kms \
--storage-class=STANDARD_IA \
"/var/lib/harrow/redis/dump.rdb" \
"s3://{{ vault.backup.aws_s3_bucket_name }}/$dirname/dump.rdb"
}
function backup_postgres {
dir=$(mktemp -d)
file=$(date --iso-8601=minutes)/{{ shared.postgresql.production.name }}.sqlc
trap 'rm -rf "$dir"' EXIT
/usr/bin/pg_dump \
--username {{ shared.postgresql.production.username }} \
-Fc {{ shared.postgresql.production.name }} > "$file"
/usr/bin/aws s3 cp \
--sse=aws:kms \
"$dir/$file" \
"s3://{{ vault.backup.aws_s3_bucket_name }}/postgresql-hetzner/$file"
}
case "$1" in
postgres)
backup_postgres
;;
redis)
backup_redis
;;
op-logs)
backup_op_logs
;;
*)
echo $"Usage: $0 {postgres|redis|op-logs}"
exit 1
esac
@@ -0,0 +1,65 @@
#!/bin/bash -e
# vim: ai ts=2 sw=2 et sts=2 ft=sh
# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=sh
export AWS_ACCESS_KEY_ID={{ vault.backup.aws_access_key_id }}
export AWS_SECRET_ACCESS_KEY={{ vault.backup.aws_secret_access_key }}
export AWS_DEFAULT_REGION={{ vault.backup.aws_region }}
function restore_op_logs {
/usr/bin/aws s3 sync \
--sse=aws:kms \
--storage-class=STANDARD_IA \
"s3://{{ vault.backup.aws_s3_bucket_name }}/operation-logs/" \
/var/lib/harrow/op-logs/
}
function restore_redis {
dirname=redis-hetzner/$1
/usr/bin/aws s3 cp \
--sse=aws:kms \
--storage-class=STANDARD_IA \
"s3://{{ vault.backup.aws_s3_bucket_name }}/$dirname/dump.rdb" \
"/var/lib/harrow/redis/dump.rdb"
}
function restore_postgres {
dir=$(mktemp -d)
file=$1/{{ shared.postgresql.production.name }}.sqlc
/usr/bin/aws s3 cp \
--sse=aws:kms \
"s3://{{ vault.backup.aws_s3_bucket_name }}/postgresql-hetzner/$file"
"$dir"
/usr/bin/pg_restore \
--username {{ shared.postgresql.production.username }}\
--clean \
-Fc {{ shared.postgresql.production.name}} "$dir/$file"
}
function restore{
echo Stopping Redis \& Postgresql
systemctl stop redis postgresql
restore_op_logs
restore_redis
restore_postgres
echo Starting Redis \& Postgresql
systemctl start redis postgresql
}
if [ "$#" -ne 1 ]; then
echo "needs one parameter, the date to restore from, e.g $0 $(date --iso-8601=minutes)"
exit 1
fi
read -p "Restore is destructive ARE YOU SURE?!!?!" -n 1 -r
echo # (optional) move to a new line
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
[[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 # handle exits from shell or function but don't exit interactive shell
restore
fi
@@ -1 +1,3 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM695hcEUFBvdQA4iHlRiTw30ThhSi96wxvWzXTgToap leehambley@harrow.io
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDOc/jabxS+w4oHl5V1Mk/RwoT0jv7sLNaARIt1o+1mw2Fmy3rN7wzcykXnuynkSS0MFiPqYoQ1JyqlMt+7QirsDxKJPcQIVPVWsXsfgsps79wZXZkV0qorUGv1vmNLiSo0HTCubd4lK9lqSjuElCPu8VrQwex+z1OlL7uo6NrMVmcANER1fx0GLUzTWZXAppRZaQ0msV/a1zYJopL1en3IMml1mozfl/w4qv/LuiVaS4xR/jEb1msEvCrkJXYm9p+WOgLqCayWRVd36aiqlQBy1X/3h68V+bvgTp+KmdZfNCDbLFcHBk6J20B1rPgQ6gUuwh+BP5pcbH3QkLG55xw9
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDaBMl8eRZu7BorK5xCr6RzTs2KqYh30bWggx3sLtP8mQIrrwCZG6BV7CgDM3G+cnl2DU90wm9oT6ZBAZQUvyil3cIzm25lrVbEVHj7lBa1lyf/15EKTE2wft1hlT1YR09V7mmwynVdTyULICC2fJRsFFAsI6zWsj/ztJYfqlvu2FMKjOFNALuSufRE5wnuae/oCK0Zx13VoNfha2xz2Ol677aBtkA5j2wpDGyJakVjVggbXuJQzyBIqNxpdCg1AcoRuGwqCGo2zyFv46duk6CwwiUd1kf9+k2fI0VdlkVKfxv5dqClO4HAXwgSLwccmHoYFn2hLuDgDATDloeGc2cz
@@ -0,0 +1,51 @@
---
- name: ensure that fluentd deps are installed
apt:
name: "{{ item }}"
with_items:
- ruby
- ruby-dev
- build-essential
- name: install fluentd and plugins for system
gem:
name: "{{ item.name }}"
version: "{{ item.version }}"
state: present
with_items:
- { name: 'oj', version: '3.3.9'}
- { name: 'fluentd', version: '0.14.22'}
- { name: 'fluent-plugin-elasticsearch', version: '1.10.2'}
- name: make fluentd config dir
file:
path: /etc/fluent
state: directory
- name: write config file for fluentd (nice and simple)
copy:
dest: /etc/fluent/fluent.conf
content: |
<source>
@type exec
command /bin/journalctl -f -n 0 -o json
tag systemd
format json
</source>
<match **>
@type elasticsearch
user {{ vault.basicauth.username }}
password {{ vault.basicauth.password }}
hosts elasticsearch.metrics-01.prod.harrow.io
scheme https
logstash_format true
reload_connections false
buffer_type memory
flush_interval 60
retry_limit 17
retry_wait 1.0
num_threads 1
</match>
@@ -71,6 +71,15 @@
- postfix
- email-routing
- name: "unset incorrect {{ item }}"
lineinfile:
path: /etc/postfix/main.cf
regexp: "{{item}}"
state: absent
with_items:
- "^relayhost \\=$"
- "^inet_interfaces \\= localhost$"
- name: "set relayhost"
blockinfile:
content: "relayhost = [{{ harrow.email_routing.sasl.host }}]:{{ harrow.email_routing.sasl.port }}"
Oops, something went wrong.

0 comments on commit 0a36b40

Please sign in to comment.