diff --git a/lima/roles/install_prerequisites/tasks/main.yaml b/lima/roles/install_prerequisites/tasks/main.yaml index 0b1133a8..5dfe7549 100644 --- a/lima/roles/install_prerequisites/tasks/main.yaml +++ b/lima/roles/install_prerequisites/tasks/main.yaml @@ -20,7 +20,6 @@ name: '{{ item }}' state: present with_items: - - python3-pip - pgedge-postgresql18 - pgedge-postgresql17 - pgedge-postgresql16 @@ -36,16 +35,11 @@ - pgedge-postgresql18-contrib - pgedge-postgresql17-contrib - pgedge-postgresql16-contrib + - pgedge-patroni - pgedge-pgbackrest - - pgedge-python3-psycopg2 - which - wget - git -- name: Install python modules - pip: - state: present - name: - - "patroni[etcd,jsonlogger]==4.1.0" - name: Install etcdctl and etcdutl ansible.builtin.unarchive: src: "{{ etcd_download_url }}" diff --git a/server/internal/config/config.go b/server/internal/config/config.go index b2f34bce..55bdf9c0 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -132,7 +132,7 @@ func (s SystemD) validate() []error { var defaultSystemD = SystemD{ PgBackRestPath: "/usr/bin/pgbackrest", - PatroniPath: "/usr/local/bin/patroni", + PatroniPath: "/usr/bin/patroni", } type HTTP struct { diff --git a/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/enable_fast_basebackup.yaml b/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/enable_fast_basebackup.yaml index fc667ed1..61be3b70 100644 --- a/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/enable_fast_basebackup.yaml +++ b/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/enable_fast_basebackup.yaml @@ -2,7 +2,7 @@ name: storefront-n1-689qacsi namespace: /patroni/ scope: storefront:n1 log: - type: json + type: plain level: INFO static_fields: database_id: storefront @@ -31,25 +31,25 @@ etcd3: protocol: https username: instance.storefront-n1-689qacsi password: password - cacert: /var/lib/pgsql/storefront-n1-689qacsi/certificates/etcd/ca.crt - cert: /var/lib/pgsql/storefront-n1-689qacsi/certificates/etcd/client.crt - key: /var/lib/pgsql/storefront-n1-689qacsi/certificates/etcd/client.key + cacert: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/etcd/ca.crt + cert: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/etcd/client.crt + key: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/etcd/client.key postgresql: authentication: superuser: username: pgedge sslmode: verify-full - sslkey: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/superuser.key - sslcert: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/superuser.crt - sslrootcert: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/ca.crt + sslkey: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/superuser.key + sslcert: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/superuser.crt + sslrootcert: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/ca.crt replication: username: patroni_replicator sslmode: verify-full - sslkey: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/replication.key - sslcert: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/replication.crt - sslrootcert: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/ca.crt + sslkey: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/replication.key + sslcert: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/replication.crt + sslrootcert: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/ca.crt connect_address: storefront-n1-689qacsi.storefront-database:5432 - data_dir: /var/lib/pgsql/storefront-n1-689qacsi/data/pgdata + data_dir: /var/lib/pgsql/18/storefront-n1-689qacsi/data/pgdata listen: "*:5432" parameters: archive_command: /bin/true @@ -76,9 +76,9 @@ postgresql: spock.include_ddl_repset: "on" spock.save_resolutions: "on" ssl: "on" - ssl_ca_file: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/ca.crt - ssl_cert_file: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/server.crt - ssl_key_file: /var/lib/pgsql/storefront-n1-689qacsi/certificates/postgres/server.key + ssl_ca_file: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/ca.crt + ssl_cert_file: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/server.crt + ssl_key_file: /var/lib/pgsql/18/storefront-n1-689qacsi/certificates/postgres/server.key track_io_timing: "on" wal_log_hints: "on" wal_sender_timeout: 5s diff --git a/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/minimal_systemd.yaml b/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/minimal_systemd.yaml index 4059171a..99338cbb 100644 --- a/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/minimal_systemd.yaml +++ b/server/internal/orchestrator/common/golden_test/TestPatroniConfigGenerator/minimal_systemd.yaml @@ -2,7 +2,7 @@ name: storefront-n1-689qacsi namespace: /patroni/ scope: storefront:n1 log: - type: json + type: plain level: INFO static_fields: database_id: storefront diff --git a/server/internal/orchestrator/common/patroni_config_generator.go b/server/internal/orchestrator/common/patroni_config_generator.go index 7d6463f7..40e814e5 100644 --- a/server/internal/orchestrator/common/patroni_config_generator.go +++ b/server/internal/orchestrator/common/patroni_config_generator.go @@ -37,6 +37,8 @@ type PatroniConfigGenerator struct { FQDN string `json:"fqdn"` // InstanceID is this instance's ID. InstanceID string `json:"instance_id"` + // LogType sets the Patroni log type. + LogType patroni.LogType `json:"log_type"` // MemoryBytes is the amount of memory that is allocated for this instance. // This is used for the tunable Postgres parameters. MemoryBytes uint64 `json:"memory_bytes,omitempty"` @@ -79,6 +81,8 @@ type PatroniConfigGeneratorOptions struct { // FQDN is the fully-qualified domain name for this instance. This name must // be reachable by sibling instances within the Spock node. FQDN string + // LogType sets the Patroni log type. + LogType patroni.LogType // OrchestratorParameters are additional parameters to be provided by the // orchestrator implementation. OrchestratorParameters map[string]any @@ -124,6 +128,7 @@ func NewPatroniConfigGenerator(opts PatroniConfigGeneratorOptions) *PatroniConfi EtcdCertsDir: opts.Paths.Instance.EtcdCertificates(), FQDN: opts.FQDN, InstanceID: opts.Instance.InstanceID, + LogType: opts.LogType, MemoryBytes: memoryBytes, NodeName: opts.Instance.NodeName, NodeOrdinal: opts.Instance.NodeOrdinal, @@ -242,9 +247,13 @@ func (p *PatroniConfigGenerator) log() *patroni.Log { if p.TenantID != nil { fields["tenant_id"] = *p.TenantID } + logType := p.LogType + if logType == "" { + logType = patroni.LogTypePlain + } return &patroni.Log{ - Type: utils.PointerTo(patroni.LogTypeJson), + Type: &logType, Level: utils.PointerTo(patroni.LogLevelInfo), StaticFields: &fields, } diff --git a/server/internal/orchestrator/common/patroni_config_generator_test.go b/server/internal/orchestrator/common/patroni_config_generator_test.go index 25a5ce5c..aabfee36 100644 --- a/server/internal/orchestrator/common/patroni_config_generator_test.go +++ b/server/internal/orchestrator/common/patroni_config_generator_test.go @@ -58,6 +58,7 @@ func TestPatroniConfigGenerator(t *testing.T) { HostCPUs: 4, HostMemoryBytes: 1024 * 1024 * 1024 * 8, FQDN: "storefront-n1-689qacsi.storefront-database", + LogType: patroni.LogTypeJson, OrchestratorParameters: map[string]any{ "shared_preload_libraries": "pg_stat_statements,snowflake,spock,postgis-3", }, @@ -115,6 +116,7 @@ func TestPatroniConfigGenerator(t *testing.T) { HostCPUs: 4, HostMemoryBytes: 1024 * 1024 * 1024 * 8, FQDN: "storefront-n1-689qacsi.storefront-database", + LogType: patroni.LogTypeJson, OrchestratorParameters: map[string]any{ "shared_preload_libraries": "pg_stat_statements,snowflake,spock,postgis-3", }, @@ -172,6 +174,7 @@ func TestPatroniConfigGenerator(t *testing.T) { HostCPUs: 4, HostMemoryBytes: 1024 * 1024 * 1024 * 8, FQDN: "storefront-n1-689qacsi.storefront-database", + LogType: patroni.LogTypeJson, OrchestratorParameters: map[string]any{ "shared_preload_libraries": "pg_stat_statements,snowflake,spock,postgis-3", }, @@ -230,6 +233,7 @@ func TestPatroniConfigGenerator(t *testing.T) { HostCPUs: 4, HostMemoryBytes: 1024 * 1024 * 1024 * 8, FQDN: "storefront-n1-689qacsi.storefront-database", + LogType: patroni.LogTypeJson, OrchestratorParameters: map[string]any{ "shared_preload_libraries": "pg_stat_statements,snowflake,spock,postgis-3", }, @@ -295,7 +299,7 @@ func TestPatroniConfigGenerator(t *testing.T) { Instance: database.Paths{BaseDir: "/var/lib/pgsql/18/storefront-n1-689qacsi"}, Host: database.Paths{BaseDir: "/var/lib/pgsql/18/storefront-n1-689qacsi"}, PgBackRestPath: "/usr/bin/pgbackrest", - PatroniPath: "/usr/local/bin/patroni", + PatroniPath: "/usr/bin/patroni", }, }, etcdHosts: []string{"i-0123456789abcdef.ec2.internal:2379"}, @@ -333,10 +337,10 @@ func TestPatroniConfigGenerator(t *testing.T) { PatroniPort: 8888, PostgresPort: 5432, Paths: database.InstancePaths{ - Instance: database.Paths{BaseDir: "/var/lib/pgsql/storefront-n1-689qacsi"}, - Host: database.Paths{BaseDir: "/var/lib/pgsql/storefront-n1-689qacsi"}, + Instance: database.Paths{BaseDir: "/var/lib/pgsql/18/storefront-n1-689qacsi"}, + Host: database.Paths{BaseDir: "/var/lib/pgsql/18/storefront-n1-689qacsi"}, PgBackRestPath: "/usr/bin/pgbackrest", - PatroniPath: "/usr/local/bin/patroni", + PatroniPath: "/usr/bin/patroni", }, }, etcdHosts: []string{ diff --git a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/cpu_limit.service b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/cpu_limit.service index aa30b0b5..7bea69b1 100644 --- a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/cpu_limit.service +++ b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/cpu_limit.service @@ -4,7 +4,7 @@ After=syslog.target network.target [Service] Type=simple User=postgres -ExecStart=/usr/local/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml +ExecStart=/usr/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml ExecReload=/bin/kill -s HUP $MAINPID KillMode=process TimeoutSec=30 diff --git a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/fractional_cpu_limit.service b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/fractional_cpu_limit.service index 8d3e2ee9..86d40308 100644 --- a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/fractional_cpu_limit.service +++ b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/fractional_cpu_limit.service @@ -4,7 +4,7 @@ After=syslog.target network.target [Service] Type=simple User=postgres -ExecStart=/usr/local/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml +ExecStart=/usr/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml ExecReload=/bin/kill -s HUP $MAINPID KillMode=process TimeoutSec=30 diff --git a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/memory_max.service b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/memory_max.service index b9e78ed2..ff28515b 100644 --- a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/memory_max.service +++ b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/memory_max.service @@ -4,7 +4,7 @@ After=syslog.target network.target [Service] Type=simple User=postgres -ExecStart=/usr/local/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml +ExecStart=/usr/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml ExecReload=/bin/kill -s HUP $MAINPID KillMode=process TimeoutSec=30 diff --git a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/minimal.service b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/minimal.service index 859019f8..f94ab1af 100644 --- a/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/minimal.service +++ b/server/internal/orchestrator/systemd/golden_test/TestUnitOptions/PatroniUnitOptions/minimal.service @@ -4,7 +4,7 @@ After=syslog.target network.target [Service] Type=simple User=postgres -ExecStart=/usr/local/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml +ExecStart=/usr/bin/patroni /var/lib/pgsql/18/storefront-n1-689qacsi/configs/patroni.yaml ExecReload=/bin/kill -s HUP $MAINPID KillMode=process TimeoutSec=30 diff --git a/server/internal/orchestrator/systemd/unit_options_test.go b/server/internal/orchestrator/systemd/unit_options_test.go index 0e431bf1..45872744 100644 --- a/server/internal/orchestrator/systemd/unit_options_test.go +++ b/server/internal/orchestrator/systemd/unit_options_test.go @@ -49,7 +49,7 @@ func TestUnitOptions(t *testing.T) { Instance: database.Paths{BaseDir: "/var/lib/pgsql/18/storefront-n1-689qacsi"}, Host: database.Paths{BaseDir: "/var/lib/pgsql/18/storefront-n1-689qacsi"}, PgBackRestPath: "/usr/bin/pgbackrest", - PatroniPath: "/usr/local/bin/patroni", + PatroniPath: "/usr/bin/patroni", } pgBinPath := "/usr/pgsql-18/bin"