Allow cortex to run in multiple services

README.md

@@ -19,6 +19,8 @@ All variables which can be overridden are stored in [defaults/main.yml](defaults
 
 | Name           | Default Value | Description                        |
 | -------------- | ------------- | -----------------------------------|
+| `cortex_all_in_one` | true | Setup Cortex in all-in-one binary mode. |
+| `cortex_services` | `{}` | Cortex services configurations. |
 | `cortex_web_listen_address` | "0.0.0.0:9009" | Address on which cortex will listen |
 | `cortex_binary_local_dir` | "" | Allows to use local packages instead of ones distributed on github. As parameter it takes a directory where `cortex` binaries are stored on host on which ansible is ran. This overrides `cortex_version` parameter |
 | `cortex_interface` | "{{ ansible_default_ipv4.interface }}" | Network adapter that cortex will be using |
@@ -66,6 +68,34 @@ Use it in a playbook as follows:
   roles:
     - cloudalchemy.cortex
 ```
+
+### Services mode
+
+You can run the different Cortex modules as separate services by setting
+`cortex_services` as a module:config map and `cortex_all_in_one` to `false`.
+```yaml
+- hosts: all
+  roles:
+    - cloudalchemy.cortex
+  vars:
+    cortex_all_in_one: false
+    cortex_services:
+      ingester:
+        server:
+            http_listen_port: 9009
+        ingester:
+            lifecycler:
+              join_after: 0
+              min_ready_duration: 0s
+              final_sleep: 0s
+              num_tokens: 512
+
+              ring:
+                kvstore:
+                  store: inmemory
+                  replication_factor: 1
+```
+
 ## Local Testing
 
 The preferred way of locally testing the role is to use Docker and [molecule](https://github.com/metacloud/molecule) (v2.x). You will have to install Docker on your system. See "Get started" for a Docker package suitable to for your system.

defaults/main.yml

@@ -1,8 +1,13 @@
 ---
+cortex_version: 1.4.0
+cortex_all_in_one: true
+
 cortex_web_listen_address: "0.0.0.0:9009"
 cortex_binary_local_dir: ''
 cortex_skip_install: false
 
+cortex_services: {}
+
 cortex_interface: "{{ ansible_default_ipv4.interface }}"
 
 cortex_config_file: 'cortex.yml.j2'
@@ -12,8 +17,6 @@ cortex_db_dir: /var/lib/cortex
 cortex_system_user: "cortex"
 cortex_system_group: "cortex"
 
-cortex_version: 1.4.0
-
 cortex_auth_enabled: false
 
 cortex_server:

handlers/main.yml

@@ -1,12 +1,35 @@
 ---
-- name: restart cortex
+- name: restart cortex all-in-one
   become: true
   systemd:
     daemon_reload: true
     name: cortex
     state: restarted
-- name: reload cortex
+  listen: restart cortex
+  when: cortex_all_in_one
+- name: restart cortex all-in-one
+  become: true
+  systemd:
+    daemon_reload: true
+    name: cortex@{{ item.key }}
+    state: restarted
+  listen: restart cortex
+  loop: "{{ cortex_services | dict2items }}"
+- name: restart cortex all-in-one
+  set_fact:
+    _cortex_service_status: started
+  listen: restart cortex
+- name: reload cortex all-in-one
   become: true
   systemd:
     name: cortex
-    state: reloaded
+    state: "{{ _cortex_service_status | default('reloaded') }}"
+  listen: reload cortex
+  when: cortex_all_in_one
+- name: reload cortex services
+  become: true
+  systemd:
+    name: cortex@{{ item.key }}
+    state: "{{ _cortex_service_status | default('reloaded') }}"
+  listen: reload cortex
+  loop: "{{ cortex_services | dict2items }}"

molecule/latest/tests/test_latest.py

@@ -1,4 +1,3 @@
-import pytest
 import os
 import testinfra.utils.ansible_runner
 

molecule/services/molecule.yml

@@ -0,0 +1,74 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: bionic
+    image: quay.io/paulfantom/molecule-systemd:ubuntu-18.04
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  - name: xenial
+    image: quay.io/paulfantom/molecule-systemd:ubuntu-16.04
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  - name: stretch
+    image: quay.io/paulfantom/molecule-systemd:debian-9
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  - name: buster
+    image: quay.io/paulfantom/molecule-systemd:debian-10
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  - name: centos7
+    image: quay.io/paulfantom/molecule-systemd:centos-7
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  - name: centos8
+    image: quay.io/paulfantom/molecule-systemd:centos-8
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    groups:
+      - python3
+  - name: fedora
+    image: quay.io/paulfantom/molecule-systemd:fedora-30
+    docker_host: "${DOCKER_HOST:-unix://var/run/docker.sock}"
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    groups:
+      - python3
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  playbooks:
+    create: ../default/create.yml
+    prepare: prepare.yml
+    converge: playbook.yml
+    destroy: ../default/destroy.yml
+  inventory:
+    group_vars:
+      python3:
+        ansible_python_interpreter: /usr/bin/python3
+scenario:
+  name: services
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: true

molecule/services/playbook.yml

@@ -0,0 +1,27 @@
+---
+- name: Run role
+  hosts: all
+  any_errors_fatal: true
+  roles:
+    - ansible-cortex
+  vars:
+    cortex_config_dir: /cortex
+    cortex_db_dir: /cortexdb
+    cortex_system_user: "vortex"
+    cortex_system_group: "vortex"
+    cortex_all_in_one: false
+    cortex_services:
+      ingester:
+        server:
+          http_listen_port: 9009
+        ingester:
+          lifecycler:
+            join_after: 0
+            min_ready_duration: 0s
+            final_sleep: 0s
+            num_tokens: 512
+
+            ring:
+              kvstore:
+                store: inmemory
+                replication_factor: 1

molecule/services/prepare.yml

@@ -0,0 +1,5 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  tasks: []

molecule/services/tests/test_services.py

@@ -0,0 +1,52 @@
+import pytest
+import os
+import yaml
+import testinfra.utils.ansible_runner
+
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+@pytest.fixture()
+def AnsibleDefaults():
+    with open("../../defaults/main.yml", 'r') as stream:
+        return yaml.load(stream)
+
+
+@pytest.mark.parametrize("dirs", [
+    "/cortex",
+    "/cortex/rules",
+    "/cortexdb"
+])
+def test_directories(host, dirs):
+    d = host.file(dirs)
+    assert d.is_directory
+    assert d.exists
+
+
+@pytest.mark.parametrize("files", [
+    "/cortex/cortex-ingester.yml",
+    "/etc/systemd/system/cortex@.service",
+    "/usr/local/bin/cortex-linux-amd64",
+])
+def test_files(host, files):
+    f = host.file(files)
+    assert f.exists
+    assert f.is_file
+
+
+def test_user(host):
+    assert host.group("vortex").exists
+    assert host.user("vortex").exists
+
+
+def test_service(host):
+    s = host.service("cortex@ingester")
+    # assert s.is_enabled
+    assert s.is_running
+
+
+def test_socket(host):
+    s = host.socket("tcp://0.0.0.0:9009")
+    assert s.is_listening

tasks/configure.yml

@@ -1,12 +1,13 @@
 ---
 - name: Copy the cortex systemd service file
   template:
-    src: cortex.service.j2
-    dest: /etc/systemd/system/cortex.service
+    src: "{{ item }}.service.j2"
+    dest: /etc/systemd/system/{{ item }}.service
     owner: root
     group: root
     mode: 0644
   notify: restart cortex
+  loop: [cortex, cortex@]
 
 - name: configure cortex
   template:
@@ -18,3 +19,16 @@
     mode: 0640
   notify:
     - reload cortex
+  when: cortex_all_in_one
+
+- name: configure cortex targets
+  copy:
+    content: "{{ item.value | to_nice_yaml(indent=2) }}"
+    dest: "{{ cortex_config_dir }}/cortex-{{ item.key }}.yml"
+    force: true
+    owner: root
+    group: "{{ cortex_system_user }}"
+    mode: 0640
+  notify:
+    - reload cortex
+  loop: "{{ cortex_services | dict2items }}"

tasks/main.yml

@@ -26,21 +26,66 @@
   tags:
     - cortex_configure
 
-- name: Ensure cortex is enabled on boot
+- name: Ensure cortex all-in-one is enabled on boot
   become: true
   systemd:
     daemon_reload: true
     name: cortex
     enabled: true
   tags:
     - cortex_run
+  when: cortex_all_in_one
 
-- name: ensure cortex service is started and enabled
+- name: ensure cortex all-in-one service is started and enabled
   become: true
   systemd:
     daemon_reload: true
     name: cortex
+    state: "{% if cortex_all_in_one %}started{% else %}stopped{%endif%}"
+    enabled: "{{ cortex_all_in_one }}"
+  tags:
+    - cortex_run
+
+- name: Ensure cortex services are disabled on boot
+  become: true
+  systemd:
+    daemon_reload: true
+    name: cortex@{{ item }}
+    enabled: false
+  tags:
+    - cortex_run
+  when: "item not in cortex_services"
+  loop: "{{ existing_cortex_units }}"
+
+- name: Ensure cortex services are stopped
+  become: true
+  systemd:
+    daemon_reload: true
+    name: cortex@{{ item }}
+    state: "stopped"
+    enabled: false
+  tags:
+    - cortex_run
+  when: "item not in cortex_services"
+  loop: "{{ existing_cortex_units }}"
+
+- name: Ensure cortex services are enabled on boot
+  become: true
+  systemd:
+    daemon_reload: true
+    name: cortex@{{ item.key }}
+    enabled: true
+  tags:
+    - cortex_run
+  loop: "{{ cortex_services | dict2items }}"
+
+- name: ensure cortex services are started and enabled
+  become: true
+  systemd:
+    daemon_reload: true
+    name: cortex@{{ item.key }}
     state: started
     enabled: true
   tags:
     - cortex_run
+  loop: "{{ cortex_services | dict2items }}"