From 56c3e5f551ed35b2337d38fbf9da7b3424e85065 Mon Sep 17 00:00:00 2001
From: Maxim Bagirov <maxim.bagirov@tantorlabs.ru>
Date: Wed, 18 Jun 2025 16:45:51 +0300
Subject: [PATCH 1/5] Changes that have been made: * replace sh certificate
 generation procedure to the ansible-way; * add scaling functionality for etcd
 nodes; * replace default DB versions; * minor changes in var names;

---
 inventory/group_vars/etcd.yml                 |   4 +-
 inventory/group_vars/keepalived.yml           |   2 +-
 inventory/group_vars/patroni.yml              |   8 +-
 inventory/group_vars/postgres_classic.yml     |   2 +-
 inventory/group_vars/tantordb.yml             |   2 +-
 pg-cluster.yaml                               |   7 +-
 roles/certificates/tasks/main.yml             | 164 +++++++++++++
 roles/certificates/vars/main.yml              |   1 +
 roles/etcd-ssl/tasks/main.yml                 |  21 --
 roles/etcd/tasks/cluster_add.yml              |  42 ++++
 roles/etcd/tasks/cluster_del.yml              |  42 ++++
 roles/etcd/tasks/cluster_state.yml            |  39 ++++
 roles/etcd/tasks/main.yml                     |  25 ++
 roles/etcd/tasks/pki.yml                      |   2 -
 roles/etcd/templates/etcd.conf.j2             |   4 +-
 .../check_scripts/chk_patroni_leader.sh       |   4 +-
 roles/keepalived/meta/argument_specs.yml      |  11 +
 roles/patroni/tasks/main.yml                  |  13 +-
 tools/etcd.conf                               |   0
 tools/etcd/.gitkeep                           |   0
 tools/etcd_preparation_template.j2            |   3 -
 tools/ssl-gen.sh                              | 221 ------------------
 22 files changed, 350 insertions(+), 267 deletions(-)
 create mode 100644 roles/certificates/tasks/main.yml
 create mode 100644 roles/certificates/vars/main.yml
 delete mode 100644 roles/etcd-ssl/tasks/main.yml
 create mode 100644 roles/etcd/tasks/cluster_add.yml
 create mode 100644 roles/etcd/tasks/cluster_del.yml
 create mode 100644 roles/etcd/tasks/cluster_state.yml
 create mode 100644 roles/keepalived/meta/argument_specs.yml
 delete mode 100644 tools/etcd.conf
 delete mode 100644 tools/etcd/.gitkeep
 delete mode 100644 tools/etcd_preparation_template.j2
 delete mode 100755 tools/ssl-gen.sh

diff --git a/inventory/group_vars/etcd.yml b/inventory/group_vars/etcd.yml
index 9abb61b..abb1c4a 100644
--- a/inventory/group_vars/etcd.yml
+++ b/inventory/group_vars/etcd.yml
@@ -8,7 +8,9 @@ etcd_data_dir: /opt/tantor/var/lib/etcd
 etcd_master_group_name: inv_etcd
 
 etcd_secure: True
-etcd_pki_dir: "{{ [ playbook_dir, 'pki-dir', hostvars[groups['inv_etcd'][0]]['ansible_hostname'] ] | path_join }}"
+etcd_bin_path: "/opt/tantor/usr/bin/etcdctl"
+etcd_conf_dir: "/opt/tantor/var/lib/etcd/pg-cluster.pki"
+etcd_pki_dir: "{{playbook_dir}}/pki-dir/{{ ansible_inventory_sources[0] | basename | regex_replace('\\.(ini|yml|yaml)$', '') }}"
 etcd_pki_key_suffix: -key.pem
 etcd_pki_cert_suffix: .pem
 
diff --git a/inventory/group_vars/keepalived.yml b/inventory/group_vars/keepalived.yml
index d0996c4..7a88e65 100644
--- a/inventory/group_vars/keepalived.yml
+++ b/inventory/group_vars/keepalived.yml
@@ -2,5 +2,5 @@
 keepalived_package_version: ""
 
 # Cluster variables
-cluster_vip_1: "xxx.xxx.xxx.xxx"
+cluster_vip_1: ""
 vip_interface: "{{ ansible_default_ipv4.interface }}"  # interface name (ex. "ens32")
diff --git a/inventory/group_vars/patroni.yml b/inventory/group_vars/patroni.yml
index f9038b5..8f55224 100644
--- a/inventory/group_vars/patroni.yml
+++ b/inventory/group_vars/patroni.yml
@@ -4,7 +4,7 @@ pg_configurator_package_version: ""
 patroni_pg_exists: false #**
 patroni_pg_port: 5432
 
-patroni_config_dir: /opt/tantor/etc/patroni
+patroni_config_dir: "/opt/tantor/etc/patroni"
 patroni_system_user: postgres
 patroni_system_group: postgres
 
@@ -63,9 +63,9 @@ patroni_etcd3_srv: ""
 patroni_etcd3_protocol: https
 patroni_etcd3_username: ""
 patroni_etcd3_password: ""
-patroni_etcd3_cacert: "/opt/tantor/etc/patroni/ca.pem"
-patroni_etcd3_cert: "/opt/tantor/etc/patroni/{{ inventory_hostname }}.pem"
-patroni_etcd3_key: "/opt/tantor/etc/patroni/{{ inventory_hostname }}-key.pem"
+patroni_etcd3_cacert: "{{ patroni_config_dir }}/ca.pem"
+patroni_etcd3_cert: "{{ patroni_config_dir }}/{{ inventory_hostname }}.pem"
+patroni_etcd3_key: "{{ patroni_config_dir }}/{{ inventory_hostname }}-key.pem"
 
 # https://patroni.readthedocs.io/en/latest/SETTINGS.html#bootstrap-configuration
 # dcs (Dynamic Configuration settings): This section will be written into /<namespace>/<scope>/config of the
diff --git a/inventory/group_vars/postgres_classic.yml b/inventory/group_vars/postgres_classic.yml
index 5a507e7..0bf58f1 100644
--- a/inventory/group_vars/postgres_classic.yml
+++ b/inventory/group_vars/postgres_classic.yml
@@ -3,4 +3,4 @@
 config_system_locale: 'ru_RU.UTF-8'
 config_system_language: 'en_US.UTF-8'
 postgresql_debian_gpg_key: "https://www.postgresql.org/media/keys/ACCC4CF8.asc"
-major_version: 15
\ No newline at end of file
+major_version: 17
\ No newline at end of file
diff --git a/inventory/group_vars/tantordb.yml b/inventory/group_vars/tantordb.yml
index 4a20172..966cd93 100644
--- a/inventory/group_vars/tantordb.yml
+++ b/inventory/group_vars/tantordb.yml
@@ -1,4 +1,4 @@
 ---
 
-major_version: 15
+major_version: 17
 edition: "be"
\ No newline at end of file
diff --git a/pg-cluster.yaml b/pg-cluster.yaml
index 11f6178..991e4c8 100644
--- a/pg-cluster.yaml
+++ b/pg-cluster.yaml
@@ -11,13 +11,13 @@
   roles:
     - prepare_nodes
 
-- name: Generage SSL certs for etcd
+- name: Generage certs
   hosts: localhost
   connection: local
   become: true
-  tags: etcd
+  tags: certificates
   roles:
-    - role: etcd-ssl
+    - role: certificates
 
 - name: Install etcd
   hosts: inv_etcd
@@ -89,6 +89,7 @@
   tags: keepalived
   vars_files:
     - 'inventory/group_vars/etcd.yml'
+    - 'inventory/group_vars/patroni.yml'
     - 'inventory/group_vars/keepalived.yml'
   roles:
     - keepalived
diff --git a/roles/certificates/tasks/main.yml b/roles/certificates/tasks/main.yml
new file mode 100644
index 0000000..3b256b0
--- /dev/null
+++ b/roles/certificates/tasks/main.yml
@@ -0,0 +1,164 @@
+- name: Generate inventory-specific certificate directory name
+  ansible.builtin.set_fact:
+    inventory_name: "{{ ansible_inventory_sources[0] | basename | regex_replace('\\.(ini|yml|yaml)$', '') }}"
+
+- name: Create inventory-specific certificate directory
+  ansible.builtin.file:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}"
+    state: directory
+    mode: '0755'
+
+- name: Get etcd nodes from inventory
+  ansible.builtin.set_fact:
+    etcd_nodes: "{{ groups['inv_etcd'] | default([]) }}"
+
+- name: Debug etcd nodes count
+  ansible.builtin.debug:
+    msg: "Found {{ etcd_nodes | length }} etcd nodes: {{ etcd_nodes }}"
+
+- name: Check if CA certificate exists
+  ansible.builtin.stat:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca.pem"
+  register: ca_cert_check
+
+- name: Check if CA key exists
+  ansible.builtin.stat:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca-key.pem"
+  register: ca_key_check
+
+- name: Generate CA private key
+  community.crypto.openssl_privatekey:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca-key.pem"
+    size: 2048
+    type: RSA
+  when: not ca_key_check.stat.exists
+
+- name: Generate CA CSR
+  community.crypto.openssl_csr:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca.csr"
+    privatekey_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca-key.pem"
+    common_name: "etcd-ca"
+    organization_name: "etcd"
+    basic_constraints:
+      - "CA:TRUE"
+    basic_constraints_critical: true
+    key_usage:
+      - keyCertSign
+      - cRLSign
+    key_usage_critical: true
+  when: not ca_cert_check.stat.exists
+
+- name: Generate CA certificate
+  community.crypto.x509_certificate:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca.pem"
+    csr_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca.csr"
+    privatekey_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca-key.pem"
+    provider: selfsigned
+  when: not ca_cert_check.stat.exists
+
+- name: Clean up CA CSR
+  ansible.builtin.file:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca.csr"
+    state: absent
+  when: not ca_cert_check.stat.exists
+
+- name: Check existing node certificates
+  ansible.builtin.stat:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}.pem"
+  register: node_cert_files_check
+  loop: "{{ etcd_nodes }}"
+
+- name: Check existing node private keys
+  ansible.builtin.stat:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}-key.pem"
+  register: node_key_files_check
+  loop: "{{ etcd_nodes }}"
+
+- name: Initialize missing certificates list
+  ansible.builtin.set_fact:
+    missing_cert_nodes: []
+
+- name: Find nodes missing certificates
+  ansible.builtin.set_fact:
+    missing_cert_nodes: "{{ missing_cert_nodes + [item.item] }}"
+  loop: "{{ node_cert_files_check.results }}"
+  when: not item.stat.exists
+
+- name: Find nodes missing private keys
+  ansible.builtin.set_fact:
+    missing_cert_nodes: "{{ missing_cert_nodes + [item.item] }}"
+  loop: "{{ node_key_files_check.results }}"
+  when: not item.stat.exists
+
+- name: Remove duplicates from missing nodes list
+  ansible.builtin.set_fact:
+    missing_cert_nodes: "{{ missing_cert_nodes | unique }}"
+
+- name: Debug missing certificates
+  ansible.builtin.debug:
+    msg: "Nodes missing certificates: {{ missing_cert_nodes }}"
+
+- name: Generate private keys for missing nodes
+  community.crypto.openssl_privatekey:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}-key.pem"
+    size: 2048
+    type: RSA
+    mode: '0644'
+  loop: "{{ missing_cert_nodes }}"
+  when: missing_cert_nodes | length > 0
+
+- name: Generate CSR for missing nodes
+  community.crypto.openssl_csr:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}.csr"
+    privatekey_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}-key.pem"
+    common_name: "{{ item }}"
+    organization_name: "etcd"
+    subject_alt_name:
+      - "DNS:{{ item }}"
+      - "DNS:{{ hostvars[item]['ansible_fqdn'] | default(item) }}"
+      - "IP:{{ hostvars[item]['ansible_default_ipv4']['address'] | default('127.0.0.1') }}"
+      - "IP:127.0.0.1"
+      - "DNS:localhost"
+    key_usage:
+      - digitalSignature
+      - keyEncipherment
+      - keyAgreement
+    extended_key_usage:
+      - serverAuth
+      - clientAuth
+  loop: "{{ missing_cert_nodes }}"
+  when: missing_cert_nodes | length > 0
+
+- name: Generate certificates for missing nodes
+  community.crypto.x509_certificate:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}.pem"
+    csr_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}.csr"
+    ownca_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca.pem"
+    ownca_privatekey_path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/ca-key.pem"
+    provider: ownca
+    mode: '0644'
+  loop: "{{ missing_cert_nodes }}"
+  when: missing_cert_nodes | length > 0
+
+- name: Clean up CSR files
+  ansible.builtin.file:
+    path: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}/{{ item }}.csr"
+    state: absent
+  loop: "{{ missing_cert_nodes }}"
+  when: missing_cert_nodes | length > 0
+
+- name: Final certificate count check
+  ansible.builtin.find:
+    paths: "{{ playbook_dir }}/pki-dir/{{ inventory_name }}"
+    patterns: "*.pem"
+    excludes: "*-key.pem,ca-key.pem"
+  register: final_cert_count
+
+- name: Display certificate generation summary
+  ansible.builtin.debug:
+    msg: |
+      Certificate generation completed:
+      - Expected certificates: {{ etcd_nodes | length + 1 }} ({{ etcd_nodes | length }} nodes + 1 CA)
+      - Generated certificates: {{ final_cert_count.matched }}
+      - etcd nodes: {{ etcd_nodes | join(', ') }}
+      - Generated for new nodes: {{ missing_cert_nodes | join(', ') if missing_cert_nodes | length > 0 else 'none' }}
diff --git a/roles/certificates/vars/main.yml b/roles/certificates/vars/main.yml
new file mode 100644
index 0000000..ed97d53
--- /dev/null
+++ b/roles/certificates/vars/main.yml
@@ -0,0 +1 @@
+---
diff --git a/roles/etcd-ssl/tasks/main.yml b/roles/etcd-ssl/tasks/main.yml
deleted file mode 100644
index 8654e5a..0000000
--- a/roles/etcd-ssl/tasks/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-
-- name: Create etcd.conf file for SSL certs
-  ansible.builtin.template:
-    src: "{{ playbook_dir + '/tools/etcd_preparation_template.j2' }}"
-    dest: "{{ playbook_dir + '/tools/etcd.conf' }}"
-    mode: "0644"
-  changed_when: false
-
-- name: Check if CA cert exists for current inventory
-  ansible.builtin.stat:
-    path: "{{ [playbook_dir, 'pki-dir', hostvars[groups['inv_etcd'][0]]['ansible_hostname'], 'ca.pem'] | path_join }}"
-  register: ca_presence
-
-- name: Generate SSL certs
-  ansible.builtin.script: "{{ playbook_dir + '/tools/ssl-gen.sh' }} {{ playbook_dir + '/tools/etcd.conf' }}"
-  args:
-    removes: "{{ playbook_dir + '/tools/etcd.conf' }}"
-    chdir: "{{ playbook_dir + '/tools' }}"
-  run_once: true # noqa: run-once[task]
-  when: ca_presence.stat.exists is false
diff --git a/roles/etcd/tasks/cluster_add.yml b/roles/etcd/tasks/cluster_add.yml
new file mode 100644
index 0000000..41588e2
--- /dev/null
+++ b/roles/etcd/tasks/cluster_add.yml
@@ -0,0 +1,42 @@
+---
+
+- name: Get current cluster members
+  when: cluster_exists
+  ansible.builtin.shell: |
+    ETCDCTL_API=3 {{ etcd_bin_path }} \
+      --endpoints=https://{{ hostvars[etcd_leader]['ansible_default_ipv4']['address'] }}:{{ etcd_port_client }} \
+      --cacert=/opt/tantor/etc/patroni/ca.pem \
+      --cert=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
+      --key=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
+      member list --write-out=json
+  register: current_members
+  delegate_to: "{{ etcd_leader }}"
+  run_once: true
+  changed_when: false
+
+- name: Parse current members
+  when: cluster_exists and current_members is defined
+  ansible.builtin.set_fact:
+    existing_member_names: >-
+      {{
+        (current_members.stdout | from_json).members |
+        map(attribute='name') |
+        list
+      }}
+  run_once: true
+
+- name: Add new etcd members to cluster
+  when:
+    - cluster_exists
+    - ansible_hostname not in existing_member_names
+    - inventory_hostname != etcd_leader
+  ansible.builtin.shell: |
+    ETCDCTL_API=3 {{ etcd_bin_path }} \
+      --endpoints=https://{{ hostvars[etcd_leader]['ansible_default_ipv4']['address'] }}:{{ etcd_port_client }} \
+      --cacert=/opt/tantor/etc/patroni/ca.pem \
+      --cert=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
+      --key=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
+      member add {{ ansible_hostname }} \
+      --peer-urls=https://{{ ansible_default_ipv4.address }}:{{ etcd_port_peer }}
+  delegate_to: "{{ etcd_leader }}"
+  register: member_add_result
diff --git a/roles/etcd/tasks/cluster_del.yml b/roles/etcd/tasks/cluster_del.yml
new file mode 100644
index 0000000..5b99a98
--- /dev/null
+++ b/roles/etcd/tasks/cluster_del.yml
@@ -0,0 +1,42 @@
+---
+
+- name: Identify nodes to remove
+  ansible.builtin.set_fact:
+    nodes_to_remove: >-
+      {{
+        existing_member_names | default([]) |
+        difference(groups['inv_etcd'] | map('extract', hostvars, 'ansible_hostname') | list)
+      }}
+  when: cluster_exists
+  run_once: true
+
+- name: Get member HEX IDs for removal
+  when: cluster_exists and nodes_to_remove | length > 0
+  ansible.builtin.shell: |
+    ETCDCTL_API=3 {{ etcd_bin_path }} \
+      --endpoints=https://{{ hostvars[etcd_leader]['ansible_default_ipv4']['address'] }}:{{ etcd_port_client }} \
+      --cacert={{ etcd_conf_dir }}/ca.pem \
+      --cert={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
+      --key={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
+      member list | grep "{{ item }}" | cut -d',' -f1
+  register: member_hex_ids_to_remove
+  delegate_to: "{{ etcd_leader }}"
+  with_items: "{{ nodes_to_remove }}"
+  failed_when: false
+
+- name: Remove etcd members from cluster
+  when: 
+    - cluster_exists 
+    - nodes_to_remove | length > 0
+    - item.stdout != ""
+    - item.rc == 0
+  ansible.builtin.shell: |
+    ETCDCTL_API=3 {{ etcd_bin_path }} \
+      --endpoints=https://{{ hostvars[etcd_leader]['ansible_default_ipv4']['address'] }}:{{ etcd_port_client }} \
+      --cacert={{ etcd_conf_dir }}/ca.pem \
+      --cert={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
+      --key={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
+      member remove {{ item.stdout }}
+  delegate_to: "{{ etcd_leader }}"
+  with_items: "{{ member_hex_ids_to_remove.results }}"
+  run_once: true
diff --git a/roles/etcd/tasks/cluster_state.yml b/roles/etcd/tasks/cluster_state.yml
new file mode 100644
index 0000000..124b257
--- /dev/null
+++ b/roles/etcd/tasks/cluster_state.yml
@@ -0,0 +1,39 @@
+---
+
+- name: Check existing etcd cluster members
+  ansible.builtin.shell: |
+    ETCDCTL_API=3 {{ etcd_bin_path }} \
+      --endpoints=https://{{ ansible_default_ipv4.address }}:{{ etcd_port_client }} \
+      --cacert={{ etcd_conf_dir }}/ca.pem \
+      --cert={{ etcd_conf_dir }}/{{ ansible_hostname }}.pem \
+      --key={{ etcd_conf_dir }}/{{ ansible_hostname }}-key.pem \
+      member list --write-out=json
+  register: etcd_member_list
+  failed_when: false
+  changed_when: false
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups['inv_etcd'] }}"
+  run_once: true
+
+- name: Parse existing cluster state
+  ansible.builtin.set_fact:
+    active_etcd_nodes: >-
+      {{
+        etcd_member_list.results |
+        selectattr('rc', 'equalto', 0) |
+        map(attribute='item') |
+        list
+      }}
+    cluster_exists: >-
+      {{
+        etcd_member_list.results |
+        selectattr('rc', 'equalto', 0) |
+        list | length > 0
+      }}
+  run_once: true
+
+- name: Determine cluster leader for management operations
+  ansible.builtin.set_fact:
+    etcd_leader: "{{ active_etcd_nodes | first }}"
+  when: active_etcd_nodes | length > 0
+  run_once: true
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index dfecaa4..fb2fd4e 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -72,10 +72,35 @@
   with_items:
     - '{{ etcd_cluster_pki_dir }}'
 
+- name: Add tasks for checking ETCD cluster
+  ansible.builtin.include_tasks: cluster_state.yml
+
+- name: Add nodes into ETCD cluster
+  ansible.builtin.include_tasks: cluster_add.yml
+
+- name: Remove nodes into ETCD cluster
+  ansible.builtin.include_tasks: cluster_del.yml
+
 - name: Add tasks for secure ETCD connection
   ansible.builtin.include_tasks: pki.yml
   when: etcd_secure | bool
 
+
+- name: Set etcd configuration parameters
+  ansible.builtin.set_fact:
+    etcd_initial_cluster_state: >-
+      {%- if cluster_exists and ansible_hostname not in (existing_member_names | default([])) -%}
+      existing
+      {%- else -%}
+      new
+      {%- endif -%}
+    etcd_use_initial_token: >-
+      {%- if cluster_exists and ansible_hostname not in (existing_member_names | default([])) -%}
+      false
+      {%- else -%}
+      true
+      {%- endif -%}
+
 - name: Copy etcd configuration
   ansible.builtin.template:
     src: etcd.conf.j2
diff --git a/roles/etcd/tasks/pki.yml b/roles/etcd/tasks/pki.yml
index f7d5425..421aed4 100644
--- a/roles/etcd/tasks/pki.yml
+++ b/roles/etcd/tasks/pki.yml
@@ -1,7 +1,5 @@
 ---
 - name: Install keys/certs
-  become: true
-  become_user: root
   with_items:
     - f: '{{ etcd_pki_key_src }}'
       d: '{{ etcd_pki_key_dest }}'
diff --git a/roles/etcd/templates/etcd.conf.j2 b/roles/etcd/templates/etcd.conf.j2
index 7848ac7..1d2639b 100644
--- a/roles/etcd/templates/etcd.conf.j2
+++ b/roles/etcd/templates/etcd.conf.j2
@@ -22,8 +22,10 @@ ETCD_ADVERTISE_CLIENT_URLS="{{etcd_scheme}}{{etcd_address_public}}:{{etcd_port_c
 ETCD_INITIAL_ADVERTISE_PEER_URLS="{{etcd_scheme}}{{etcd_address_cluster}}:{{etcd_port_peer}}"
 {% endif %}
 ETCD_INITIAL_CLUSTER="{{etcd_cluster}}"
-ETCD_INITIAL_CLUSTER_STATE="new"
+ETCD_INITIAL_CLUSTER_STATE="{{ etcd_initial_cluster_state }}"
+{% if etcd_use_initial_token %}
 ETCD_INITIAL_CLUSTER_TOKEN="{{etcd_initial_cluster_token}}"
+{% endif %}
 # if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
 #ETCD_DISCOVERY=""
 #ETCD_DISCOVERY_SRV=""
diff --git a/roles/keepalived/check_scripts/chk_patroni_leader.sh b/roles/keepalived/check_scripts/chk_patroni_leader.sh
index 707ba98..ee452b7 100644
--- a/roles/keepalived/check_scripts/chk_patroni_leader.sh
+++ b/roles/keepalived/check_scripts/chk_patroni_leader.sh
@@ -1,13 +1,13 @@
 #!/bin/bash
 
-NODENAME=$(cat /opt/tantor/etc/patroni/{{ inventory_hostname }}.yml | grep -E "^name:" | cut -d: -f2 | tr -d '[:blank:]')
+NODENAME=$(cat {{ patroni_config_dir }}/{{ inventory_hostname }}.yml | grep -E "^name:" | cut -d: -f2 | tr -d '[:blank:]')
 
 if [[ -z "$NODENAME" ]]; then
    echo "Nodename is blank!"
    exit 1
 fi
 
-PATRONICTL_OUT=$(/opt/tantor/usr/bin/patronictl -c /opt/tantor/etc/patroni/{{ inventory_hostname }}.yml list --format json)
+PATRONICTL_OUT=$(/opt/tantor/usr/bin/patronictl -c {{ patroni_config_dir }}/{{ inventory_hostname }}.yml list --format json)
 
 if [[ -z "$PATRONICTL_OUT" ]]; then
    echo "No patronictl output!"
diff --git a/roles/keepalived/meta/argument_specs.yml b/roles/keepalived/meta/argument_specs.yml
new file mode 100644
index 0000000..9b70b13
--- /dev/null
+++ b/roles/keepalived/meta/argument_specs.yml
@@ -0,0 +1,11 @@
+---
+
+argument_specs:
+  main:
+    short_description: Validating pg_cluster variables for keepalived role
+    description: Checks presence and types of all required variables for pg_cluster deployment
+    options:
+      cluster_vip_1:
+        type: "str"
+        required: true
+        format: ipv4
diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml
index c726da2..692ae4a 100644
--- a/roles/patroni/tasks/main.yml
+++ b/roles/patroni/tasks/main.yml
@@ -356,13 +356,13 @@
       --min-conns=200 \
       --max-conns=200 \
       --common-conf \
-      --output-file-name=/opt/tantor/etc/patroni/postgresql.json
+      --output-file-name='{{ patroni_config_dir }}/postgresql.json'
   args:
-    creates: /opt/tantor/etc/patroni/postgresql.json
+    creates: '{{ patroni_config_dir }}/postgresql.json'
 
 - name: Get content of remote file
   ansible.builtin.slurp:
-    src: /opt/tantor/etc/patroni/postgresql.json
+    src: '{{ patroni_config_dir }}/postgresql.json'
   register: remote_yaml
 
 - name: Interpret remote file content as yaml
@@ -437,13 +437,13 @@
   become_user: root
   with_items:
     - f: '{{ etcd_pki_key_src }}'
-      d: /opt/tantor/etc/patroni
+      d: '{{ patroni_config_dir }}'
       m: '0400'
     - f: '{{ etcd_pki_cert_src }}'
-      d: /opt/tantor/etc/patroni
+      d: '{{ patroni_config_dir }}'
       m: '0600'
     - f: '{{ etcd_pki_ca_cert_src }}'
-      d: /opt/tantor/etc/patroni
+      d: '{{ patroni_config_dir }}'
       m: '0600'
   ansible.builtin.copy:
     src: '{{ item.f }}'
@@ -451,6 +451,7 @@
     owner: postgres
     group: postgres
     mode: '{{ item.m }}'
+  notify: Restart patroni
 
 - name: Ensure patroni is running
   ansible.builtin.systemd_service:
diff --git a/tools/etcd.conf b/tools/etcd.conf
deleted file mode 100644
index e69de29..0000000
diff --git a/tools/etcd/.gitkeep b/tools/etcd/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/tools/etcd_preparation_template.j2 b/tools/etcd_preparation_template.j2
deleted file mode 100644
index 7779c25..0000000
--- a/tools/etcd_preparation_template.j2
+++ /dev/null
@@ -1,3 +0,0 @@
-{% for host in groups['inv_etcd'] %}
-{{ hostvars[host]['ansible_hostname']}},{{ hostvars[host]['ansible_host']}},{{ hostvars[host]['ansible_hostname']}},{{ hostvars[host]['ansible_host']}}
-{% endfor %}
\ No newline at end of file
diff --git a/tools/ssl-gen.sh b/tools/ssl-gen.sh
deleted file mode 100755
index d29da5e..0000000
--- a/tools/ssl-gen.sh
+++ /dev/null
@@ -1,221 +0,0 @@
-#!/bin/bash
-# https://github.com/portworx/cfssl-certs
-# generate certs for etcd or consul using CloudFlare's cfssl [201611.22MeV]
-# https://www.digitalocean.com/community/tutorials/how-to-secure-your-coreos-cluster-with-tls-ssl-and-firewall-rules
-# https://github.com/cloudflare/cfssl
-
-# config file is in the form
-# DNS name, internal AWS DNS, internal AWS IP, external AWS IP
-
-if [ "`echo $PATH | grep /usr/local/bin`" == "" ]; then PATH=$PATH:/usr/local/bin; fi
-
-function USAGE {
-	cat <<-EOFUSAGE
-
-    generate SSL certs for etcd or consul using config file
-
-    Usage: ${0##*/} -dhn <CONFIG_FILE>
-
-    where
-
-    -h  this help function
-    -n  dry-run (don't generate certs but loop through config file)
-EOFUSAGE
-	exit 2
-}
-
-# parse arguments
-OPT_d=1; OPT_n=1; QUIET="--quiet"
-while getopts ":dhn" OPT; do
-	case ${OPT} in
-
-	d)  OPT_D=0      # DEBUG mode
-		QUIET=""
-		;;
-
-	n)  OPT_n=0      # dry-run mode
-		;;
-
-	h|?) USAGE
-		;;
-
-	esac
-done
-shift $((OPTIND-1)) # move to next argument after options
-
-[ $# -lt 1 ] && echo "?config filename missing" && USAGE
-[ ! -e ${1} ] && echo "?config file missing" && USAGE
-
-## defaults
-#
-conf_file=${1}
-tmp=${1##*/}		# longest occurance of slash from back
-conf_name=${tmp%%.*}	# longest occurance of dot from front
-cur_d=$(pwd)
-
-ssl_folder=$(cat ${conf_file} | head -n1 | cut -d "," -f1)
-mkdir -p ../pki-dir/"${ssl_folder}"
-
-ca=ca
-ca_csr="/tmp/${ca}-csr.json"
-ca_config="/tmp/${ca}-config.json"
-
-# rsa/2048...ecdsa is a smaller key but takes 10x to encode/decode
-key_algo=rsa
-key_size=2048
-cert_expire=`expr 50 \* 365 \* 24`	# hours in 50 years
-
-C="RU"
-L=""
-O=""
-ST="Moscow"
-OU=""
-
-[ ! -e /usr/local/bin/ ] && mkdir -P /usr/local/bin
-case $(uname) in
-	Darwin)
-		if [ ! -e  /usr/local/bin/cfssl ]; then
-			curl -s -L -o /usr/local/bin/cfssl \
-				https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64
-		fi
-		if [ ! -e  /usr/local/bin/cfssljson ]; then
-			curl -s -L -o /usr/local/bin/cfssljson \
-				https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64
-		fi
-		if [ ! -e  /usr/local/bin/mkbundle ]; then
-			curl -s -L -o /usr/local/bin/mkbundle \
-				https://pkg.cfssl.org/R1.2/mkbundle_darwin-amd64
-		fi
-		;;
-	Linux)
-		if [ ! -e  /usr/local/bin/cfssl ]; then
-			curl -s -L -o /usr/local/bin/cfssl \
-				https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
-		fi
-		if [ ! -e  /usr/local/bin/cfssljson ]; then
-			curl -s -L -o /usr/local/bin/cfssljson \
-				https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
-		fi
-		if [ ! -e  /usr/local/bin/mkbundle ]; then
-			curl -s -L -o /usr/local/bin/mkbundle \
-				https://pkg.cfssl.org/R1.2/mkbundle_linux-amd64
-		fi
-		;;
-	*)
-		echo "OS $(uname) not supported"
-		exit 2
-		;;
-esac
-chmod +x /usr/local/bin/{cfssl,cfssljson,mkbundle}
-
-[ ! -e ${conf_name} ] && mkdir -p ${conf_name}
-/bin/cp ${conf_file} ${conf_name}/
-cd ${conf_name}
-
-[ $OPT_n -eq 1 ] && cat <<-EOFCACONFIG > ${ca_config}
-{
-    "signing": {
-        "default": {
-            "expiry": "${cert_expire}h"
-        },
-        "profiles": {
-            "server": {
-                "expiry": "${cert_expire}h",
-                "usages": [
-                    "signing",
-                    "key encipherment",
-                    "server auth"
-                ]
-            },
-            "client": {
-                "expiry": "${cert_expire}h",
-                "usages": [
-                    "signing",
-                    "key encipherment",
-                    "client auth"
-                ]
-            },
-            "client-server": {
-                "expiry": "${cert_expire}h",
-                "usages": [
-                    "signing",
-                    "key encipherment",
-                    "server auth",
-                    "client auth"
-                ]
-            }
-        }
-    }
-}
-
-EOFCACONFIG
-
-[ $OPT_n -eq 1 ] && cat <<-EOFCACSR > ${ca_csr}
-{
-    "CN": "${conf_name}",
-    "key": {
-        "algo": "${key_algo}",
-        "size": ${key_size}
-    },
-    "names": [
-        {
-            "C": "${C}",
-            "L": "${L}",
-            "O": "${O}",
-            "ST": "${ST}",
-            "OU": "${OU}"
-        }
-    ]
-}
-
-EOFCACSR
-
-# from https://github.com/coreos/docs/blob/master/os/generate-self-signed-certificates.md
-# these certs work for connecting between nodes [201608.25MeV]
-# this code works for using the ca_csr above or in-line
-# works for etcdctl if you use server cert, key, and CA key.
-echo "================================================== generating self-signed CA cert+key"
-if [ ${OPT_n} -eq 1 ]; then
-	cfssl gencert -initca ${ca_csr} | cfssljson -bare ${ca}
-	# mv ${ca}-key.pem ${ca}.key
-	chmod 600 ${ca}-key.pem
-	echo -n "checking ${ca}.pem for crlsign..."
-	openssl verify -purpose crlsign -CAfile ${ca}.pem ${ca}.pem
-	openssl x509 -in ${ca}.pem -text -noout
-fi
-
-# config file is in the form: DNS-name,internal-AWS-DNS,internal-AWS-IP,external-AWS-IP
-for host in $(cat ${conf_file}) ; do
-	#h=$(echo ${host} | sed -e 's/,/","/g' -e 's/^/"/' -e 's/$/"/')
-	h=$(echo ${host} | sed -e 's/,/","/g')
-	fqdn=$(echo ${host} | awk -F"," '{print $1}') # use 1st entry for server name
-	s=${fqdn%%.*} # remove longest occurance of dot from back
-	
-    # NOTE: CN must match CN of CA key
-	f=$(printf '{"CN":"%s","hosts":["%s"],"key":{"algo":"%s","size":%s}}' ${conf_name} ${h}  ${key_algo} ${key_size})
-	echo "================================================== generating ${s} certs+keys"
-	if [ $OPT_n -eq 1 ]; then
-	    echo ${f} | cfssl gencert -ca=${ca}.pem -ca-key=${ca}-key.pem -config=${ca_config} \
-		    -hostname="${host}" -profile=client-server - | cfssljson -bare ${s}
-	    echo -n "verifying with ${ca}.pem for sslserver..."
-	    openssl verify -purpose sslserver -CAfile ${ca}.pem ${s}.pem
-	    echo -n "verifying with ${ca}.pem for sslclient..."
-	    openssl verify -purpose sslclient -CAfile ${ca}.pem ${s}.pem
-    	openssl x509 -in ${s}.pem -text -noout
-	fi
-	# mv ${s}-key.pem ${s}.key
-	chmod 600 ${s}-key.pem
-done
-
-rm -f *.csr ${conf_file}
-rm -f ${ca_csr}* ${ca_config}*
-#echo "================================================== bundling all certs"
-#if [ $OPT_n -eq 1 ]; then
-#    mkbundle -f ${conf_name}.crt .
-#    openssl x509 -in ${conf_name}.crt -text -noout
-#fi
-
-[ "*.pem" != "" ] && chmod 644 *.pem
-cd ${cur_d}
-mv -v **/*.pem ../pki-dir/"${ssl_folder}"
-exit

From a25ae40f7fbee75936a355c1bf15ac53a4960be5 Mon Sep 17 00:00:00 2001
From: Maxim Bagirov <maxim.bagirov@tantorlabs.ru>
Date: Wed, 18 Jun 2025 17:06:48 +0300
Subject: [PATCH 2/5] Add info into README file

---
 README.md | 51 +++++++++++++++++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index c28d673..8f447ac 100644
--- a/README.md
+++ b/README.md
@@ -3,8 +3,8 @@
 ## Project Structure
 
 ```
-|-- pg-cluster.yaml			# Main playbook
-|-- pki-dir				# Certificates generated using ssl-gen.sh
+|-- pg-cluster.yaml			            # Main playbook
+|-- pki-dir				                # Folder that store generated certs
 |   |-- .gitkeep
 |-- README.md
 |-- inventory
@@ -17,17 +17,25 @@
 |   |   |-- prepare_nodes.yml
 |   |-- my_inventory
 |-- roles
-|   |-- etcd					# Role that installs etcd-tantor-all package
+|   |-- certificates					# Role that generate TLS certs for etcd and patroni
+|   |   |-- tasks
+|   |   |   `-- main.yml
+|   |   |-- vars
+|   |   |   `-- main.yml
+|   |-- etcd					        # Role that installs etcd-tantor-all package
 |   |   |-- handlers
 |   |   |   `-- main.yml
 |   |   |-- tasks
 |   |   |   |-- main.yml
 |   |   |   |-- pki.yml
+|   |   |   |-- cluster_add.yml
+|   |   |   |-- cluster_del.yml
+|   |   |   |-- cluster_state.yml
 |   |   |   `-- systemd.yml
 |   |   |-- templates
 |   |   |   |-- etcd.conf.j2
 |   |   |   `-- etcd-tantor.service.j2
-|   |-- haproxy					# Role that installs haproxy-tantor-all package
+|   |-- haproxy					        # Role that installs haproxy-tantor-all package
 |   |   |-- handlers
 |   |   |   `-- main.yml
 |   |   |-- tasks
@@ -38,21 +46,25 @@
 |   |   |-- check_scripts
 |   |   |   `-- chk_patroni_leader.sh
 |   |   |-- handlers
-|   |   |   `-- main.yml 					# 
+|   |   |   `-- main.yml 
+|   |   |-- meta
+|   |   |   `-- argument_specs.yml
 |   |   |-- tasks
 |   |   |   `-- main.yml
 |   |   `-- templates
 |   |       `-- keepalived.conf.j2
-|   |-- patroni					# Role that installs patroni-tantor-all package
+|   |-- patroni					        # Role that installs patroni-tantor-all package
 |   |   |-- handlers
 |   |   |   `-- main.yml
 |   |   |-- tasks
 |   |   |   `-- main.yml
 |   |   `-- templates
-|   |       |-- patroni.service.j2
+|   |       |-- patroni_custom_bootstrap_script.sh.j2
+|   |       |-- patroni-tantor.service.j2
 |   |       |-- patroni-watchdog.service.j2
-|   |       `-- patroni.yml.j2
-|   |-- pgbouncer				# Role that installs pgbouncer-tantor-all package
+|   |       |-- patroni.yml.j2
+|   |       `-- walg.json.j2
+|   |-- pgbouncer				        # Role that installs pgbouncer-tantor-all package
 |   |   |-- handlers
 |   |   |   `-- main.yml 
 |   |   |-- sql
@@ -62,22 +74,25 @@
 |   |   `-- templates
 |   |       |-- pgbouncer.ini.j2
 |   |       `-- pgbouncer.service.j2
-|   |-- postgres_classic			# Role that installs postgresql package
+|   |-- postgres_classic			    # Role that installs postgresql package
 |   |   `-- tasks
 |   |       `-- main.yml
-|   |-- postgres_tantordb			# Role that installs tantor-server package
+|   |-- postgres_tantordb			    # Role that installs tantor-server package
 |   |   `-- tasks
 |   |       `-- main.yml
-|   `-- prepare_nodes			# Role for installing basic utils
+|   `-- prepare_nodes			        # Role for installing basic utils
 |       `-- handlers
 |           `-- main.yml
 |       `-- tasks
-|           `-- main.yml
+|           |-- main.yml
+|           |-- debian.yml
+|           `-- rhel.yml
 |-- tools
-|   |-- etcd
-|   |-- etcd.conf
-|   |-- pg_configurator.py
-|   `-- ssl-gen.sh
+|   `-- pg_cluster_backend			    # In progress
+|       |-- conf 
+|       |-- log 
+|       |-- psc 
+|       `-- pg_cluster_backend.py 
 ```
 
 ![Architecture](pg_cluster_architechture.png)
@@ -90,7 +105,7 @@ The following text will present examples of commands to be entered in the termin
 ## Requirements
 
 Playbook requires the following component's version to be installed:
-* Ansible >= 2.9.10
+* Ansible >= 2.9.10 (with collections community.general, community.postgresql, community.crypto)
 * Python3 (with pip module) >= 3.10.0
 * psycopg2 >= 2.5.1 (it's recommended to install via pip)
 * packaging >= 24 (it's recommended to install via pip)

From cefde12ea8cc65dad4165b94fa404db15265dd0c Mon Sep 17 00:00:00 2001
From: Maxim Bagirov <maxim.bagirov@tantorlabs.ru>
Date: Thu, 19 Jun 2025 12:09:56 +0300
Subject: [PATCH 3/5] Fix etcd_configuration directory issue

---
 roles/etcd/tasks/cluster_add.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/roles/etcd/tasks/cluster_add.yml b/roles/etcd/tasks/cluster_add.yml
index 41588e2..463375b 100644
--- a/roles/etcd/tasks/cluster_add.yml
+++ b/roles/etcd/tasks/cluster_add.yml
@@ -5,9 +5,9 @@
   ansible.builtin.shell: |
     ETCDCTL_API=3 {{ etcd_bin_path }} \
       --endpoints=https://{{ hostvars[etcd_leader]['ansible_default_ipv4']['address'] }}:{{ etcd_port_client }} \
-      --cacert=/opt/tantor/etc/patroni/ca.pem \
-      --cert=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
-      --key=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
+      --cacert={{ etcd_conf_dir }}/ca.pem \
+      --cert={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
+      --key={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
       member list --write-out=json
   register: current_members
   delegate_to: "{{ etcd_leader }}"
@@ -33,9 +33,9 @@
   ansible.builtin.shell: |
     ETCDCTL_API=3 {{ etcd_bin_path }} \
       --endpoints=https://{{ hostvars[etcd_leader]['ansible_default_ipv4']['address'] }}:{{ etcd_port_client }} \
-      --cacert=/opt/tantor/etc/patroni/ca.pem \
-      --cert=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
-      --key=/opt/tantor/etc/patroni/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
+      --cacert={{ etcd_conf_dir }}/ca.pem \
+      --cert={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}.pem \
+      --key={{ etcd_conf_dir }}/{{ hostvars[etcd_leader]['ansible_hostname'] }}-key.pem \
       member add {{ ansible_hostname }} \
       --peer-urls=https://{{ ansible_default_ipv4.address }}:{{ etcd_port_peer }}
   delegate_to: "{{ etcd_leader }}"

From 77cfb7f7f1423e4e4b8979f4e3d3f421b0961a1b Mon Sep 17 00:00:00 2001
From: Maxim Bagirov <maxim.bagirov@tantorlabs.ru>
Date: Thu, 19 Jun 2025 12:50:32 +0300
Subject: [PATCH 4/5] Minor changes in README and versioning

---
 README.md                                 | 14 +++++++-------
 inventory/group_vars/postgres_classic.yml |  2 +-
 inventory/group_vars/tantordb.yml         |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 8f447ac..a799163 100644
--- a/README.md
+++ b/README.md
@@ -241,7 +241,7 @@ By default, the playbook does not attempt to connect to Tantor repositories and
 * wal-g-tantor-all
 * tantor DBMS
 
-Pay attention to last point from the list above. Tantor package should match environment that is used during playbook launch. For example if you want to install ``tantor-be-server-15`` DBMS using command ``ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=tantordb edition=be major_version=15" pg-cluster.yaml -K`` make sure that package ``tantor-be-server-15`` is available in your local repository.  
+Pay attention to last point from the list above. Tantor package should match environment that is used during playbook launch. For example if you want to install ``tantor-be-server-16`` DBMS using command ``ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=tantordb edition=be major_version=16" pg-cluster.yaml -K`` make sure that package ``tantor-be-server-16`` is available in your local repository.  
 
 If the playbook is run in an environment with internet access, you can leverage the most up-to-date components included in the solution. To do this, add the flag ``add_nexus_repo=true`` and provide the connection details for the repositories in the file ``inventory/group_vars/prepare_nodes.yml``.
 
@@ -252,7 +252,7 @@ There are several options to run Ansible: with the option to install TantorDB or
 Use the following command to install TantorDB:
 
 ```bash
-ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=tantordb edition=be major_version=15" pg-cluster.yaml -K
+ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=tantordb edition=be major_version=16" pg-cluster.yaml -K
 ```
 
 Use the following command to install the PostgreSQL DBMS:
@@ -267,7 +267,7 @@ In the commands above, replace the value of the ``major_version`` parameter with
 
 It's possible to launch the playbook with external internet access.
 ```bash
-ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=tantordb edition=be major_version=15 add_nexus_repo=true" pg-cluster.yaml -K
+ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=tantordb edition=be major_version=16 add_nexus_repo=true" pg-cluster.yaml -K
 ```
 In that case, make sure that connection details are provided in the file ``inventory/group_vars/prepare_nodes.yml``.
 
@@ -281,10 +281,10 @@ Below you can find some common commands for working with the software products i
 # on NODE_1
 e_host=(
   /opt/tantor/usr/bin/etcdctl
-  --endpoints=https://<HOST_1_IP>:2379,https://<HOST_2_IP>:2379,https://<HOST_N_IP>:2379
-  --cacert=/opt/tantor/etc/patroni/ca.pem
-  --cert=/opt/tantor/etc/patroni/<NODE1_HOSTNAME>.pem  
-  --key=/opt/tantor/etc/patroni/<NODE1_HOSTNAME>-key.pem
+  --endpoints=https://$(hostname -I | awk '{print $1}'):2379
+  --cacert=/opt/tantor/var/lib/etcd/pg-cluster.pki/ca.pem
+  --cert=/opt/tantor/var/lib/etcd/pg-cluster.pki/$(hostname).pem  
+  --key=/opt/tantor/var/lib/etcd/pg-cluster.pki/$(hostname)-key.pem
 )
 
 # list etcd members
diff --git a/inventory/group_vars/postgres_classic.yml b/inventory/group_vars/postgres_classic.yml
index 0bf58f1..8d3ae37 100644
--- a/inventory/group_vars/postgres_classic.yml
+++ b/inventory/group_vars/postgres_classic.yml
@@ -3,4 +3,4 @@
 config_system_locale: 'ru_RU.UTF-8'
 config_system_language: 'en_US.UTF-8'
 postgresql_debian_gpg_key: "https://www.postgresql.org/media/keys/ACCC4CF8.asc"
-major_version: 17
\ No newline at end of file
+major_version: 16
\ No newline at end of file
diff --git a/inventory/group_vars/tantordb.yml b/inventory/group_vars/tantordb.yml
index 966cd93..2b5d4c1 100644
--- a/inventory/group_vars/tantordb.yml
+++ b/inventory/group_vars/tantordb.yml
@@ -1,4 +1,4 @@
 ---
 
-major_version: 17
+major_version: 16
 edition: "be"
\ No newline at end of file

From 59f7d424349127bb8a113d19be0f84c37a1b15bd Mon Sep 17 00:00:00 2001
From: Maxim Bagirov <maxim.bagirov@tantorlabs.ru>
Date: Fri, 4 Jul 2025 13:22:30 +0300
Subject: [PATCH 5/5] Add info regarding component maintenance into README
 file; Add fail condition for all plays

---
 README.md       | 4 ++++
 ansible.cfg     | 3 ++-
 pg-cluster.yaml | 9 +++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a799163..b21e6ef 100644
--- a/README.md
+++ b/README.md
@@ -271,6 +271,10 @@ ansible-playbook -i inventory/my_inventory -u admin_user -e "postgresql_vendor=t
 ```
 In that case, make sure that connection details are provided in the file ``inventory/group_vars/prepare_nodes.yml``.
 
+## Component maintenance
+
+The playbook supports both full and partial updates for most components. Each role includes a variable that defines the desired version of a component (e.g., the variable ``pg_configurator_package_version`` corresponds to the ``pg-configurator-tantor-all`` component). These variables are defined in the ``inventory/group_vars`` YAML files. On the first run, the latest versions of the components will be installed. If you need to install a specific version, simply set the appropriate variable and run the playbook again.
+
 ## HOW TO
 
 Below you can find some common commands for working with the software products included in the ``pg_cluster`` solution. Note that the commands and their result may differ depending on the software versions used.
diff --git a/ansible.cfg b/ansible.cfg
index 887f5d6..4f9d1bd 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -4,4 +4,5 @@ command_timeout = 60
 [defaults]
 timeout = 60
 log_path = ansible.log
-host_key_checking = False # отключение проверки fingerprint
\ No newline at end of file
+host_key_checking = False # отключение проверки fingerprint
+fail_on_error = true
\ No newline at end of file
diff --git a/pg-cluster.yaml b/pg-cluster.yaml
index 991e4c8..31ccb14 100644
--- a/pg-cluster.yaml
+++ b/pg-cluster.yaml
@@ -10,6 +10,7 @@
     - 'inventory/group_vars/prepare_nodes.yml'
   roles:
     - prepare_nodes
+  any_errors_fatal: true
 
 - name: Generage certs
   hosts: localhost
@@ -18,6 +19,7 @@
   tags: certificates
   roles:
     - role: certificates
+  any_errors_fatal: true
 
 - name: Install etcd
   hosts: inv_etcd
@@ -27,6 +29,7 @@
     - 'inventory/group_vars/etcd.yml'
   roles:
     - role: etcd
+  any_errors_fatal: true
 
 - name: Install PostgreSQL Tantordb
   hosts: inv_pg
@@ -37,6 +40,7 @@
     - 'inventory/group_vars/tantordb.yml'
   roles:
     - postgres_tantordb
+  any_errors_fatal: true
 
 - name: Install PostgreSQL Classic
   hosts: inv_pg
@@ -47,6 +51,7 @@
     - 'inventory/group_vars/postgres_classic.yml'
   roles:
     - postgres_classic
+  any_errors_fatal: true
 
 - name: Install Patroni
   hosts: inv_pg
@@ -59,6 +64,7 @@
     - 'inventory/group_vars/postgres_classic.yml'
   roles:
     - patroni
+  any_errors_fatal: true
 
 - name: Install PGBouncer
   hosts: inv_pg
@@ -70,6 +76,7 @@
     - 'inventory/group_vars/patroni.yml'
   roles:
     - pgbouncer
+  any_errors_fatal: true
 
 - name: Install Haproxy
   hosts: inv_pg
@@ -82,6 +89,7 @@
     - 'inventory/group_vars/pgbouncer.yml'
   roles:
     - haproxy
+  any_errors_fatal: true
 
 - name: Install Keepalived
   hosts: inv_keepalived
@@ -93,3 +101,4 @@
     - 'inventory/group_vars/keepalived.yml'
   roles:
     - keepalived
+  any_errors_fatal: true