Extensible configuration (#90)

Allow for configuration of arbitrary node and cluster configuration items.
N.B. Further work to be done on idempotence and integration of TLS. #74 and #86 will need some rework

.gitignore

@@ -8,3 +8,4 @@ ansible/playbooks/tls/ca
 ansible/playbooks/tls/certs
 hosts.ini
 .DS_Store
+.idea/*

README.md

@@ -43,6 +43,23 @@ You can pass the following variables as `-e var=value`:
 * `grafana_admin_pass=<password_here>`: Configure Grafana's admin user's password
 * `ephemeral_disk`: Enable filesystem check for attached disk, useful when using attached disks in instances with ephemeral OS disks (i.e Azure L Series). This allows a filesystem repair at boot time and ensures that the drive is remounted automatically after a reboot. Default `false` 
 
+The Redpanda role defines a number of default values which can be overriden using `--extra-vars`. For a full list see the [role defaults file](ansible/playbooks/roles/redpanda_broker/defaults/main.yml).
+
+You can also specify any available Redpanda configuration value (or set of values) by passing a JSON dictionary as an Ansible extra-var. These values will be spliced with the calculated configuration and only override those values that you specify.
+There are two sub-dictionaries that you can specify, `redpanda.cluster` and `redpanda.node`. Check the Redpanda docs for the available [Cluster configuration properties](https://docs.redpanda.com/docs/platform/reference/cluster-properties/) and [Node configuration properties](https://docs.redpanda.com/docs/platform/reference/node-properties/).
+
+An example overriding specific properties would be as follows:
+
+```commandline
+ansible-playbook ansible/playbooks/provision-node.yml -i hosts.ini  --extra-vars '{ "redpanda": 
+{"cluster":
+  { "auto_create_topics_enabled": "true"},
+ "node":
+  { "developer_mode": "false"}
+  }
+}'
+```
+
 2. Use `rpk` & standard Kafka tools to produce/consume from the Redpanda cluster & access the Grafana installation on the monitor host.
 * The Grafana URL is http://&lt;grafana host&gt;:3000/login
 

ansible/playbooks/provision-node.yml

@@ -2,8 +2,8 @@
 
 - import_playbook: prepare-data-dir.yml
 
-- import_playbook: install-redpanda.yml
+- hosts: redpanda
+  roles:
+    - redpanda_broker
 
-- import_playbook: start-redpanda.yml
-
-- import_playbook: deploy-prometheus-grafana.yml
+- import_playbook: deploy-prometheus-grafana.yml

ansible/playbooks/roles/redpanda_broker/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+
+redpanda_mode: production
+redpanda_admin_api_port: 9644
+redpanda_kafka_port: 9092
+redpanda_rpc_port: 33145
+
+redpanda_key_file: /etc/redpanda/certs/node.key
+redpanda_cert_file: /etc/redpanda/certs/node.crt
+redpanda_truststore_file: /etc/redpanda/certs/truststore.pem

ansible/playbooks/roles/redpanda_broker/tasks/install-redpanda.yml

@@ -0,0 +1,27 @@
+---
+- name: add the redpanda repo
+  shell: |
+    curl -1sLf https://packages.vectorized.io/sMIXnoa7DK12JW4A/redpanda/cfg/setup/bash.deb.sh | sudo -E bash
+  args:
+    warn: no
+  when: ansible_os_family == 'Debian'
+
+- name: add the redpanda repo
+  shell: |
+    curl -1sLf https://packages.vectorized.io/sMIXnoa7DK12JW4A/redpanda/cfg/setup/bash.rpm.sh | sudo -E bash
+  args:
+    warn: no
+  when: ansible_os_family == 'RedHat'
+
+- name: install redpanda from repository
+  package:
+    name:
+      - redpanda
+    state: present
+    update_cache: yes
+
+- name: set data dir file perms
+  file:
+    path: /var/lib/redpanda/data
+    owner: redpanda
+    group: redpanda

ansible/playbooks/roles/redpanda_broker/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+
+- name: Install Redpanda
+  include_tasks: install-redpanda.yml
+
+- name: Start Redpanda
+  include_tasks: start-redpanda.yml

ansible/playbooks/roles/redpanda_broker/tasks/start-redpanda.yml

@@ -0,0 +1,59 @@
+- name: ensure /etc/redpanda exists
+  file:
+    path: /etc/redpanda
+    state: directory
+    mode: 0755
+
+- name: Reset configuration
+  set_fact:
+    configuration: {}
+
+- name: Generate configurations
+  set_fact:
+    configuration: "{{ configuration | combine(lookup('template', custom_config_template.template) | from_yaml, recursive=True) }}"
+  loop: "{{ custom_config_templates }}"
+  loop_control:
+    loop_var: custom_config_template
+  when: custom_config_template.condition | default(True)
+
+- name: Merge with user-provided overrides (via redpanda variable)
+  set_fact:
+    configuration: "{{ configuration | combine(redpanda | default({}), recursive=True) }}"
+
+- name: Generate bootstrap
+  ansible.builtin.template:
+    src: "bootstrap.yml"
+    dest: "/etc/redpanda/.bootstrap.yaml"
+    owner: redpanda
+    group: redpanda
+
+- name: Generate Node config
+  ansible.builtin.template:
+    src: "redpanda.yml"
+    dest: "/etc/redpanda/redpanda.yaml"
+    owner: redpanda
+    group: redpanda
+
+- name: configure redpanda
+  shell: |
+    set -e
+    sudo -u redpanda rpk redpanda mode production
+    {% if hostvars[groups['redpanda'][0]].id == hostvars[inventory_hostname].id %}
+    sudo -u redpanda rpk redpanda config bootstrap \
+      --id {{ hostvars[inventory_hostname].id }} \
+      --self {{ hostvars[inventory_hostname].private_ip }}
+    {% else %}
+    sudo -u redpanda rpk redpanda config bootstrap \
+      --id {{ hostvars[inventory_hostname].id }} \
+      --self {{ hostvars[inventory_hostname].private_ip }} \
+      --ips {{ groups["redpanda"] | map('extract', hostvars) | map(attribute='private_ip') | join(',') }}
+    {% endif %}
+
+- name: start redpanda-tuner
+  systemd:
+    name: redpanda-tuner
+    state: started
+- name: start redpanda
+  systemd:
+    name: redpanda
+    state: started

ansible/playbooks/roles/redpanda_broker/templates/bootstrap.yml

@@ -0,0 +1 @@
+{{ configuration.cluster | to_nice_yaml }}

ansible/playbooks/roles/redpanda_broker/templates/configs/defaults.j2

@@ -0,0 +1,31 @@
+{% set address_attribute = 'inventory_hostname' if (advertise_public_ips | d() | bool) else 'private_ip' %}
+cluster:
+  rpc_server_tcp_recv_buf: 65536
+  enable_rack_awareness: {{ true if rack is defined and rack != -1 else false }}
+node:
+  cluster_id: redpanda
+  organization: redpanda-test
+  redpanda:
+    node_id: "{{ id }}"
+    data_directory: "/var/lib/redpanda/data"
+    advertised_kafka_api:
+    - address: {{ hostvars[inventory_hostname][address_attribute] }}
+      port: {{ redpanda_kafka_port }}
+    advertised_rpc_api:
+      address: {{ hostvars[inventory_hostname].private_ip }}
+      port: {{ redpanda_rpc_port }}
+    rpc_server:
+      address: {{ hostvars[inventory_hostname].private_ip }}
+      port: {{ redpanda_rpc_port }}
+    rack: {{ rack | default('null') }}
+  rpk:
+    admin_api:
+      addresses:
+{% for host in groups["redpanda"] %}
+      - {{ hostvars[host][address_attribute] }}:{{ redpanda_admin_api_port }}
+{% endfor %}
+    kafka_api:
+      brokers:
+{% for host in groups["redpanda"] %}
+      - {{ hostvars[host][address_attribute] }}:{{ redpanda_kafka_port }}
+{% endfor %}

ansible/playbooks/roles/redpanda_broker/templates/configs/tls.j2

@@ -0,0 +1,27 @@
+node:
+  redpanda:
+    admin_api_tls:
+      enabled: true,
+      require_client_auth: false,
+      key_file: "{{ redpanda_key_file }}"
+      cert_file: "{{ redpanda_cert_file }}"
+      truststore_file: "{{ redpanda_truststore_file }}"
+    kafka_api_tls:
+      enabled: true
+      require_client_auth: false
+      key_file: "{{ redpanda_key_file }}"
+      cert_file: "{{ redpanda_cert_file }}"
+      truststore_file: "{{ redpanda_truststore_file }}"
+    rpc_server_tls:
+      enabled: true,
+      require_client_auth: false,
+      key_file: "{{ redpanda_key_file }}"
+      cert_file: "{{ redpanda_cert_file }}"
+      truststore_file: "{{ redpanda_truststore_file }}"
+  rpk:
+    admin_api:
+      tls:
+        truststore_file: "{{ redpanda_truststore_file }}"
+    kafka_api:
+      tls:
+        truststore_file: "{{ redpanda_truststore_file }}"

ansible/playbooks/roles/redpanda_broker/templates/redpanda.yml

@@ -0,0 +1 @@
+{{ configuration.node | to_nice_yaml }}

ansible/playbooks/roles/redpanda_broker/vars/main.yml

@@ -0,0 +1,6 @@
+---
+
+custom_config_templates:
+  - template: configs/defaults.j2
+  - template: configs/tls.j2
+    condition: "{{ tls | default(False) | bool }}"