From a62882a8ae0f9784d155689cdcca4ddbdef1c7ce Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Fri, 1 Jul 2016 10:16:44 -0400 Subject: [PATCH] added link support --- jobs/pgpool/spec | 7 ++++--- jobs/pgpool/templates/config/pgpool.conf | 6 +++--- jobs/postgres/spec | 18 ++++++++++++------ jobs/postgres/templates/bin/ctl | 2 +- jobs/postgres/templates/config/hba.conf | 4 ++-- jobs/postgres/templates/config/postgresql.conf | 4 ++-- jobs/smoke-tests/spec | 13 +++++-------- jobs/smoke-tests/templates/bin/run | 14 +++++++------- templates/jobs.yml | 10 ---------- 9 files changed, 36 insertions(+), 42 deletions(-) diff --git a/jobs/pgpool/spec b/jobs/pgpool/spec index 7990773..9a98b3e 100644 --- a/jobs/pgpool/spec +++ b/jobs/pgpool/spec @@ -17,6 +17,10 @@ templates: config/pcppass: config/.pcppass config/passwd: config/passwd +consumes: +- name: db + type: postgresql + properties: pgpool.debug: description: Enable extra debugging wherever possible. Intended for BOSH release maintainers. @@ -26,9 +30,6 @@ properties: description: "A list of {username: ..., password: ...} objects for defining pgpool users" default: [] - pgpool.backend.hosts: - description: A list of backend host / IPs to pool connections to. - default: [] pgpool.backend.port: description: The port that the PostgreSQL backends are listening on default: 6432 diff --git a/jobs/pgpool/templates/config/pgpool.conf b/jobs/pgpool/templates/config/pgpool.conf index 98bcc18..206924b 100644 --- a/jobs/pgpool/templates/config/pgpool.conf +++ b/jobs/pgpool/templates/config/pgpool.conf @@ -6,7 +6,7 @@ config['pcp_port'] ||= 9898 # FIXME: support for SSL key? - backends = p('pgpool.backend.hosts', []) + backends = link("db").instances if backends.empty? then raise "pgpool.backend.hosts property must define at least one postgres backend host!" end @@ -32,8 +32,8 @@ connection_cache = on reset_query_list = 'ABORT; DISCARD ALL' pool_passwd = 'passwd' -<% backends.each_with_index do |backend, i| %> -backend_host<%= i %> = <%= val(backend) %> +<% link("db").instances.each_with_index do |instance, i| %> +backend_host<%= i %> = <%= val(instance.address) %> backend_port<%= i %> = <%= val(p('pgpool.backend.port')) %> <% end %> diff --git a/jobs/postgres/spec b/jobs/postgres/spec index 592bd1e..4d3c872 100644 --- a/jobs/postgres/spec +++ b/jobs/postgres/spec @@ -17,6 +17,18 @@ templates: sql/setup-replication.sql: sql/setup-replication.sql +provides: +- name: db + type: postgresql + properties: + - postgres.config.port + - pgpool.users + +consumes: +- name: db + type: postgresql + + properties: postgres.config: description: A map of postgresql.conf configuration directives, keyed by name. @@ -79,12 +91,6 @@ properties: postgres.replication.master: description: IP address of the preferred master node (should be the 0th postgres node's IP) default: ~ - postgres.replication.pool: - description: List of all IP address of nodes in the replication pool (including the master) - default: [] - postgres.pgpool.pool: - description: List of all IP address of PGPoolII nodes that front this Postgres node - default: [] pgpool.users: description: "A list of {username: ..., password: ...} objects for defining pgpool users" diff --git a/jobs/postgres/templates/bin/ctl b/jobs/postgres/templates/bin/ctl index 8c96b67..f3c8857 100644 --- a/jobs/postgres/templates/bin/ctl +++ b/jobs/postgres/templates/bin/ctl @@ -1,7 +1,7 @@ #!/bin/bash <% -masterip = p('postgres.replication.master', '') +masterip = link("db").instances.first.address replication = p('postgres.replication.enabled', false) && masterip != '' master = replication && spec.index == 0 port = p('postgres.config.port') diff --git a/jobs/postgres/templates/config/hba.conf b/jobs/postgres/templates/config/hba.conf index 90551ed..b80bc9b 100644 --- a/jobs/postgres/templates/config/hba.conf +++ b/jobs/postgres/templates/config/hba.conf @@ -10,8 +10,8 @@ local all all md5 <% end %> # replication hosts -<% p("postgres.replication.pool", []).each do |ip| -%>host replication replication <%= ip %>/32 trust +<% link("db").instances.each do |instance| +%>host replication replication <%= instance.address %>/32 trust <% end %> # user-configured acls diff --git a/jobs/postgres/templates/config/postgresql.conf b/jobs/postgres/templates/config/postgresql.conf index 66856b9..b8a7271 100644 --- a/jobs/postgres/templates/config/postgresql.conf +++ b/jobs/postgres/templates/config/postgresql.conf @@ -1,7 +1,7 @@ # postgres main configuration <% - masterip = p('postgres.replication.master', '') - replication = p('postgres.replication.enabled', false) && masterip != '' + pgs = link("db").instances + replication = p('postgres.replication.enabled', false) && pgs.size() > 1 master = replication && spec.index == 0 config = p('postgres.config', {}) diff --git a/jobs/smoke-tests/spec b/jobs/smoke-tests/spec index abdd004..2403c2a 100644 --- a/jobs/smoke-tests/spec +++ b/jobs/smoke-tests/spec @@ -3,6 +3,11 @@ name: smoke-tests packages: - postgres - pgrt + +consumes: +- name: db + type: postgresql + templates: bin/run: bin/run helpers/ctl_setup.sh: helpers/ctl_setup.sh @@ -10,8 +15,6 @@ templates: sql/pgbench_cleanup.sql: sql/pgbench_cleanup.sql properties: - postgres.smoke-tests.target.address: - description: Hostname or IP address to run pgbench against, for generating load. postgres.smoke-tests.target.port: description: Port to run pgbench against, for generating load. default: 5432 @@ -28,12 +31,6 @@ properties: description: Port that PostgreSQL itself (not PGPoolII) is listening on, for replication health checking. default: 6432 - postgres.smoke-tests.cluster.master: - description: IP address of the write master in a replicated Postgres cluster. - postgres.smoke-tests.cluster.slaves: - description: List of IP addresses of the read slaves in a replicated Postgres cluster. - default: [] - postgres.smoke-tests.thresholds.replication-lag: description: Maximum allowable replication lag for any single read slave from master (in kb). default: 64 diff --git a/jobs/smoke-tests/templates/bin/run b/jobs/smoke-tests/templates/bin/run index a905c6b..f2babd3 100644 --- a/jobs/smoke-tests/templates/bin/run +++ b/jobs/smoke-tests/templates/bin/run @@ -1,10 +1,10 @@ #!/bin/bash <% -master = p('postgres.smoke-tests.cluster.master') +master = link("db").instances.first.address slaves = [] -p('postgres.smoke-tests.cluster.slaves', []).each do |slave| - if slave != master - slaves.push(slave) +link("db").instances.each do |instance| + if instance.address != master + slaves.push(instance.address) end end %> @@ -35,7 +35,7 @@ echo "*:*:*:<%= p('postgres.smoke-tests.target.username') %>:<%= p('postgres.smo chmod 0600 ~/.pgpass echo "INITIALIZING PGBENCH DATABASE" -pgbench -h <%= p('postgres.smoke-tests.target.address') %> \ +pgbench -h <%= master %> \ -p <%= p('postgres.smoke-tests.target.port') %> \ -U <%= p('postgres.smoke-tests.target.username') %> \ -i 2>&1 @@ -45,7 +45,7 @@ CPUS=$(grep -c ^processor /proc/cpuinfo) CLIENTS=<%= p('postgres.smoke-tests.target.clients') %> CLIENTS=$(( CLIENTS - (CLIENTS % CPUS) + CPUS )) time \ -pgbench -h <%= p('postgres.smoke-tests.target.address') %> \ +pgbench -h <%= master %> \ -p <%= p('postgres.smoke-tests.target.port') %> \ -U <%= p('postgres.smoke-tests.target.username') %> \ -c ${CLIENTS} -j ${CPUS} -T 125 -C 2>&1 & @@ -66,7 +66,7 @@ echo "pgbench exited $?" echo -psql -h <%= p('postgres.smoke-tests.target.address') %> \ +psql -h <%= master %> \ -p <%= p('postgres.smoke-tests.target.port') %> \ -U <%= p('postgres.smoke-tests.target.username') %> \ --file "/var/vcap/jobs/smoke-tests/sql/pgbench_cleanup.sql" diff --git a/templates/jobs.yml b/templates/jobs.yml index 0f26a75..c760efc 100644 --- a/templates/jobs.yml +++ b/templates/jobs.yml @@ -24,10 +24,6 @@ jobs: port: 6432 replication: enabled: true - master: (( grab jobs.postgres.networks.postgres1.static_ips[0] )) - pool: (( grab jobs.postgres.networks.postgres1.static_ips )) - pgpool: - pool: (( grab jobs.postgres.networks.postgres1.static_ips )) hba: - host all all 0.0.0.0/0 md5 - host all all ::/0 md5 @@ -46,7 +42,6 @@ jobs: - username: smoke-tests password: if-ya-got-em backend: - hosts: (( grab jobs.postgres.networks.postgres1.static_ips )) port: 6432 config: port: 5432 @@ -68,17 +63,12 @@ jobs: postgres: smoke-tests: target: - address: (( grab jobs.postgres.properties.postgres.replication.master )) port: 5432 username: smoke-tests password: if-ya-got-em backend: port: 6432 - cluster: - master: (( grab jobs.postgres.properties.postgres.replication.master )) - slaves: (( grab jobs.postgres.properties.postgres.replication.pool )) - networks: (( param "please set networks" ))