From dc03628d3e01dfe2e8f09ea57aa64662a6d01d59 Mon Sep 17 00:00:00 2001 From: Matthias Antierens Date: Tue, 22 Sep 2020 16:52:27 +0200 Subject: [PATCH 1/2] Update Cortex role --- LICENSE | 2 +- README.md | 28 ++++-- defaults/main.yml | 80 +++++++++++++++- handlers/main.yml | 5 + meta/main.yml | 2 +- molecule/alternative/playbook.yml | 9 +- .../alternative/tests/test_alternative.py | 46 ++++++++++ molecule/default/playbook.yml | 2 +- molecule/default/tests/test_default.py | 47 ++++++++++ tasks/configure.yml | 11 +++ tasks/install.yml | 91 +++++++++++++++++++ tasks/main.yml | 10 ++ tasks/preflight.yml | 52 +++++++++++ templates/cortex.service.j2 | 6 +- templates/cortex.yml.j2 | 34 +++++++ vars/main.yml | 4 +- 16 files changed, 411 insertions(+), 18 deletions(-) create mode 100644 templates/cortex.yml.j2 diff --git a/LICENSE b/LICENSE index c0f033e..c7d5c2f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2020 Pawel Krupa and Julien Pivotto +Copyright (c) 2020 Pawel Krupa, Matthias Antierens and Julien Pivotto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index ce558ea..ef4819a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## Description -Deploy [cortex](https://github.com/prometheus/cortex) using ansible. +Deploy [cortex](https://github.com/cortexproject/cortex) using ansible. ## Requirements @@ -19,7 +19,26 @@ All variables which can be overridden are stored in [defaults/main.yml](defaults | Name | Default Value | Description | | -------------- | ------------- | -----------------------------------| -| `cortex_web_listen_address` | "0.0.0.0:<>" | Address on which cortex will listen | +| `cortex_web_listen_address` | "0.0.0.0:9009" | Address on which cortex will listen | +| `cortex_binary_local_dir` | "" | Allows to use local packages instead of ones distributed on github. As parameter it takes a directory where `cortex` binaries are stored on host on which ansible is ran. This overrides `cortex_version` parameter | +| `cortex_interface` | "{{ ansible_default_ipv4.interface }}" | Network adapter that cortex will be using | +| `cortex_config_dir` | "/etc/cortex" | Default directory for the cortex config | +| `cortex_db_dir` | "/var/lib/cortex" | Path to the directory of the Cortex database | +| `cortex_system_user` | "cortex" | Default Cortex user | +| `cortex_system_group` | "cortex" | Default Cortex group | +| `cortex_version` | "1.4.0-rc.1" | The cortex package version | +| `cortex_auth_enabled` | "false" | Enables of disables the Cortex authentication | +| `cortex_server` | [From block storage example][bse] | Cortex server. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#server_config) | +| `cortex_distributor` | [From block storage example][bse] | Cortex distributor. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#distributor_config) | +| `cortex_ingester_client` | [From block storage example][bse] | Cortex ingester client. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#ingester_client_config) | +| `cortex_ingester` | [From block storage example][bse] | Cortex ingester. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#ingester_config) | +| `cortex_storage` | [From block storage example][bse] | Cortex storage. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#storage_config) | +| `cortex_blocks_storage` | [From block storage example][bse] | Cortex blocks storage. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#blocks_storage_config) | +| `cortex_compactor` | [From block storage example][bse] | Cortex compactor. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#compactor_config) | +| `cortex_frontend_worker` | [From block storage example][bse] | Cortex frontend worker. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#frontend_worker_config) | +| `cortex_ruler` | [From block storage example][bse] | Cortex ruler. Compatible with [official configuration](https://cortexmetrics.io/docs/configuration/configuration-file/#ruler_config) | + +[bse]:https://github.com/cortexproject/cortex/blob/master/docs/configuration/single-process-config-blocks.yaml ## Example @@ -31,11 +50,6 @@ Use it in a playbook as follows: roles: - cloudalchemy.cortex ``` - -### Demo site - -We provide demo site for full monitoring solution based on prometheus and grafana. Repository with code and links to running instances is [available on github](https://github.com/prometheus/demo-site) and site is hosted on [DigitalOcean](https://digitalocean.com). - ## Local Testing The preferred way of locally testing the role is to use Docker and [molecule](https://github.com/metacloud/molecule) (v2.x). You will have to install Docker on your system. See "Get started" for a Docker package suitable to for your system. diff --git a/defaults/main.yml b/defaults/main.yml index 81e972c..1aa5d77 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,2 +1,80 @@ --- -cortex_web_listen_address: "0.0.0.0:<>" +cortex_web_listen_address: "0.0.0.0:9009" +cortex_binary_local_dir: '' +cortex_skip_install: false + +cortex_interface: "{{ ansible_default_ipv4.interface }}" + +cortex_config_file: 'cortex.yml.j2' +cortex_config_dir: /etc/cortex +cortex_db_dir: /var/lib/cortex + +cortex_system_user: "cortex" +cortex_system_group: "cortex" + +cortex_version: 1.4.0-rc.1 + +cortex_auth_enabled: false + +cortex_server: + http_listen_port: 9009 + + # Configure the server to allow messages up to 100MB. + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + +cortex_distributor: + shard_by_all_labels: true + pool: + health_check_ingesters: true + +cortex_ingester_client: + grpc_client_config: + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + grpc_compression: gzip + +cortex_ingester: + lifecycler: + interface_names: [ "{{ cortex_interface }}" ] + join_after: 0 + min_ready_duration: 0s + final_sleep: 0s + num_tokens: 512 + + ring: + kvstore: + store: inmemory + replication_factor: 1 + +cortex_storage: + engine: blocks + +cortex_blocks_storage: + tsdb: + dir: /tmp/cortex/tsdb + + bucket_store: + sync_dir: /tmp/cortex/tsdb-sync + + backend: filesystem + filesystem: + dir: ./data/tsdb + +cortex_compactor: + data_dir: /tmp/cortex/compactor + sharding_ring: + kvstore: + store: inmemory + +cortex_frontend_worker: + match_max_concurrent: true + +cortex_ruler: + enable_api: true + enable_sharding: false + storage: + type: local + local: + directory: "{{ cortex_config_dir }}/rules" diff --git a/handlers/main.yml b/handlers/main.yml index e2838c5..8483ff4 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -5,3 +5,8 @@ daemon_reload: true name: cortex state: restarted +- name: reload cortex + become: true + systemd: + name: cortex + state: reloaded diff --git a/meta/main.yml b/meta/main.yml index 63be5a8..4ee1524 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,6 +1,6 @@ --- galaxy_info: - author: Julien Pivotto, Pawel Krupa + author: Matthias Antierens, Julien Pivotto description: Deploy cortex role_name: cortex license: MIT diff --git a/molecule/alternative/playbook.yml b/molecule/alternative/playbook.yml index cfb887b..5f69a4a 100644 --- a/molecule/alternative/playbook.yml +++ b/molecule/alternative/playbook.yml @@ -1,6 +1,11 @@ --- -- hosts: all +- name: Run role + hosts: all any_errors_fatal: true roles: - ansible-cortex - vars: [] + vars: + cortex_config_dir: /cortex + cortex_db_dir: /cortexdb + cortex_system_user: "vortex" + cortex_system_group: "vortex" diff --git a/molecule/alternative/tests/test_alternative.py b/molecule/alternative/tests/test_alternative.py index 3b537ad..eec13c7 100644 --- a/molecule/alternative/tests/test_alternative.py +++ b/molecule/alternative/tests/test_alternative.py @@ -1,6 +1,52 @@ import pytest import os +import yaml import testinfra.utils.ansible_runner + testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.fixture() +def AnsibleDefaults(): + with open("../../defaults/main.yml", 'r') as stream: + return yaml.load(stream) + + +@pytest.mark.parametrize("dirs", [ + "/cortex", + "/cortex/rules", + "/cortexdb" +]) +def test_directories(host, dirs): + d = host.file(dirs) + assert d.is_directory + assert d.exists + + +@pytest.mark.parametrize("files", [ + "/cortex/cortex.yml", + "/etc/systemd/system/cortex.service", + "/usr/local/bin/cortex-linux-amd64", +]) +def test_files(host, files): + f = host.file(files) + assert f.exists + assert f.is_file + + +def test_user(host): + assert host.group("vortex").exists + assert host.user("vortex").exists + + +def test_service(host): + s = host.service("cortex") + # assert s.is_enabled + assert s.is_running + + +def test_socket(host): + s = host.socket("tcp://0.0.0.0:9009") + assert s.is_listening diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml index 9f38dac..3f3583c 100644 --- a/molecule/default/playbook.yml +++ b/molecule/default/playbook.yml @@ -2,4 +2,4 @@ - hosts: all any_errors_fatal: true roles: - - ansible-cortex + - name: ansible-cortex diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py index 989363a..a24c781 100644 --- a/molecule/default/tests/test_default.py +++ b/molecule/default/tests/test_default.py @@ -1,5 +1,52 @@ +import pytest import os +import yaml import testinfra.utils.ansible_runner + testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.fixture() +def AnsibleDefaults(): + with open("../../defaults/main.yml", 'r') as stream: + return yaml.load(stream) + + +@pytest.mark.parametrize("dirs", [ + "/etc/cortex", + "/etc/cortex/rules", + "/var/lib/cortex" +]) +def test_directories(host, dirs): + d = host.file(dirs) + assert d.is_directory + assert d.exists + + +@pytest.mark.parametrize("files", [ + "/etc/cortex/cortex.yml", + "/etc/systemd/system/cortex.service", + "/usr/local/bin/cortex-linux-amd64", +]) +def test_files(host, files): + f = host.file(files) + assert f.exists + assert f.is_file + + +def test_user(host): + assert host.group("cortex").exists + assert host.user("cortex").exists + + +def test_service(host): + s = host.service("cortex") + # assert s.is_enabled + assert s.is_running + + +def test_socket(host): + s = host.socket("tcp://0.0.0.0:9009") + assert s.is_listening diff --git a/tasks/configure.yml b/tasks/configure.yml index fc15230..e50e09d 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -7,3 +7,14 @@ group: root mode: 0644 notify: restart cortex + +- name: configure cortex + template: + src: "{{ cortex_config_file }}" + dest: "{{ cortex_config_dir }}/cortex.yml" + force: true + owner: root + group: "{{ cortex_system_user }}" + mode: 0640 + notify: + - reload cortex diff --git a/tasks/install.yml b/tasks/install.yml index ba1a995..f941187 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -14,3 +14,94 @@ system: true createhome: false home: / + +- name: create cortex data directory + file: + path: "{{ cortex_db_dir }}" + state: directory + owner: "{{ cortex_system_user }}" + group: "{{ cortex_system_group }}" + mode: 0755 + + +- name: create cortex configuration directories + file: + path: "{{ item }}" + state: directory + owner: "{{ cortex_system_user }}" + group: "{{ cortex_system_group }}" + mode: 0770 + with_items: + - "{{ cortex_config_dir }}" + - "{{ cortex_config_dir }}/rules" + +- block: + - name: download cortex binary and install it + become: false + get_url: + url: "https://github.com/cortexproject/cortex/releases/download/v{{ cortex_version }}/cortex-linux-amd64" + dest: "/tmp/cortex-linux-amd64" + checksum: "sha256:{{ __cortex_checksum }}" + register: _download_archive + until: _download_archive is succeeded + retries: 5 + delay: 2 + delegate_to: localhost + check_mode: false + + - name: propagate official cortex + copy: + src: "/tmp/cortex-linux-amd64" + dest: "{{ _cortex_binary_install_dir }}/cortex-linux-amd64" + mode: 0755 + owner: root + group: root + notify: restart cortex + + when: + - cortex_binary_local_dir | length == 0 + - not cortex_skip_install + +- name: propagate locally distributed cortex + copy: + src: "{{ cortex_binary_local_dir }}/cortex-linux-amd64" + dest: "{{ _cortex_binary_install_dir }}/cortex-linux-amd64" + mode: 0755 + owner: root + group: root + when: + - cortex_binary_local_dir | length > 0 + - not cortex_skip_install + +- name: create systemd service unit + template: + src: cortex.service.j2 + dest: /etc/systemd/system/cortex.service + owner: root + group: root + mode: 0644 + notify: + - restart cortex + +- name: Install SELinux dependencies + package: + name: "{{ item }}" + state: present + with_items: "{{ cortex_selinux_packages }}" + register: _install_packages + until: _install_packages is succeeded + retries: 5 + delay: 2 + when: + - ansible_version.full is version('2.4', '>=') + - ansible_selinux.status == "enabled" + +- name: Allow cortex to bind to port in SELinux + seport: + ports: "{{ cortex_web_listen_address.split(':')[1] }}" + proto: tcp + setype: http_port_t + state: present + when: + - ansible_version.full is version('2.4', '>=') + - ansible_selinux.status == "enabled" diff --git a/tasks/main.yml b/tasks/main.yml index 5598df6..0573e75 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -34,3 +34,13 @@ enabled: true tags: - cortex_run + +- name: ensure cortex service is started and enabled + become: true + systemd: + daemon_reload: true + name: cortex + state: started + enabled: true + tags: + - cortex_run diff --git a/tasks/preflight.yml b/tasks/preflight.yml index 0a2765f..4a23efe 100644 --- a/tasks/preflight.yml +++ b/tasks/preflight.yml @@ -3,3 +3,55 @@ assert: that: ansible_service_mgr == 'systemd' msg: "This module only works with systemd" + +- name: Get systemd version + command: systemctl --version + changed_when: false + check_mode: false + register: __systemd_version + tags: + - skip_ansible_lint + +- name: Set systemd version fact + set_fact: + cortex_systemd_version: "{{ __systemd_version.stdout_lines[0].split(' ')[-1] }}" + + +- block: + - name: Get latest release + uri: + url: "https://api.github.com/repos/cortexproject/cortex/releases/latest" + method: GET + return_content: true + status_code: 200 + body_format: json + validate_certs: false + user: "{{ lookup('env', 'GH_USER') | default(omit) }}" + password: "{{ lookup('env', 'GH_TOKEN') | default(omit) }}" + no_log: "{{ not lookup('env', 'ANSIBLE_DEBUG') | bool }}" + register: _latest_release + until: _latest_release.status == 200 + retries: 5 + + - name: "Set cortex version to {{ _latest_release.json.tag_name[1:] }}" + set_fact: + cortex_version: "{{ _latest_release.json.tag_name[1:] }}" + when: + - cortex_version == "latest" + - cortex_binary_local_dir | length == 0 + - not cortex_skip_install + +- block: + - name: "Get checksum list" + set_fact: + __cortex_checksums: "{{ lookup('url', 'https://github.com/cortexproject/cortex/releases/download/v' + cortex_version + '/cortex-linux-amd64-sha-256', wantlist=True) | list }}" + run_once: true + + - name: "Get checksum for {{ go_arch }} architecture" + set_fact: + __cortex_checksum: "{{ item.split(' ')[0] }}" + with_items: "{{ __cortex_checksums }}" + delegate_to: localhost + when: + - cortex_binary_local_dir | length == 0 + - not cortex_skip_install diff --git a/templates/cortex.service.j2 b/templates/cortex.service.j2 index cc4e764..6eafa42 100644 --- a/templates/cortex.service.j2 +++ b/templates/cortex.service.j2 @@ -10,8 +10,10 @@ StartLimitIntervalSec=0 Type=simple User={{ cortex_system_user }} Group={{ cortex_system_group }} -ExecStart=/usr/local/bin/cortex -SyslogIdentifier=cortex +ExecReload=/bin/kill -HUP $MAINPID +ExecStart={{ _cortex_binary_install_dir }}/cortex-linux-amd64 -config.file={{ cortex_config_dir }}/cortex.yml +WorkingDirectory=/var/lib/cortex +SyslogIdentifier={{ cortex_system_user }} Restart=always RestartSec=5 diff --git a/templates/cortex.yml.j2 b/templates/cortex.yml.j2 new file mode 100644 index 0000000..1ea1f28 --- /dev/null +++ b/templates/cortex.yml.j2 @@ -0,0 +1,34 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +{{ ansible_managed | comment }} +# https://cortexmetrics.io/docs/ + +auth_enabled: {{ cortex_auth_enabled }} + +server: + {{ cortex_server | to_nice_yaml(indent=2) | indent(2, False) }} + +distributor: + {{ cortex_distributor | to_nice_yaml(indent=2) | indent(2, False) }} + +ingester_client: + {{ cortex_ingester_client | to_nice_yaml(indent=2) | indent(2, False) }} + +ingester: + {{ cortex_ingester | to_nice_yaml(indent=2) | indent(2, False) }} + +storage: + {{ cortex_storage | to_nice_yaml(indent=2) | indent(2, False) }} + +blocks_storage: + {{ cortex_blocks_storage | to_nice_yaml(indent=2) | indent(2, False) }} + +compactor: + {{ cortex_compactor | to_nice_yaml(indent=2) | indent(2, False) }} + +frontend_worker: + {{ cortex_frontend_worker | to_nice_yaml(indent=2) | indent(2, False) }} + +ruler: + {{ cortex_ruler | to_nice_yaml(indent=2) | indent(2, False) }} + + diff --git a/vars/main.yml b/vars/main.yml index 5bd92db..09f7323 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -7,6 +7,4 @@ go_arch_map: armv6l: 'armv6' go_arch: "{{ go_arch_map[ansible_architecture] | default(ansible_architecture) }}" - -cortex_system_user: "cortex" -cortex_system_group: "cortex" +_cortex_binary_install_dir: '/usr/local/bin' From 431e09eed6758c8aba250509cffcf37971ed43ab Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 22 Sep 2020 17:16:37 +0200 Subject: [PATCH 2/2] Fix alternate test Signed-off-by: Julien Pivotto --- templates/cortex.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/cortex.service.j2 b/templates/cortex.service.j2 index 6eafa42..5242666 100644 --- a/templates/cortex.service.j2 +++ b/templates/cortex.service.j2 @@ -12,7 +12,7 @@ User={{ cortex_system_user }} Group={{ cortex_system_group }} ExecReload=/bin/kill -HUP $MAINPID ExecStart={{ _cortex_binary_install_dir }}/cortex-linux-amd64 -config.file={{ cortex_config_dir }}/cortex.yml -WorkingDirectory=/var/lib/cortex +WorkingDirectory={{ cortex_db_dir }} SyslogIdentifier={{ cortex_system_user }} Restart=always RestartSec=5