Restore postgres infrastructure for local dev (#256)

* repair postgresql-primary state

* upgrade to postgres 11

* fixup db/user creation via salt

Vagrantfile

@@ -16,6 +16,7 @@ SERVERS = [
   "mail",
   "moin",
   "planet",
+  "postgresql",
   {:name => "postgresql-primary", :roles => ["postgresql", "postgresql-primary"]},
   {:name => "postgresql-replica", :roles => ["postgresql", "postgresql-replica"]},
   "pypy-web",

docs/services/postgresql.rst

@@ -1,27 +1,24 @@
 PostgreSQL
 ==========
 
-The Python Infrastructure offers PostgreSQL databases to services hosted in the
-Rackspace datacenter.
+The Python Infrastructure uses PostgreSQL databases to services hosted in the
+DigitalOcean datacenter.
 
+* Currently running hosted PostgreSQL 11 provided by DigitalOcean databases.
 
-* Currently running PostgreSQL 9.4
-
-* Operates a 2 node cluster with a primary node configured with streaming
-  replication to a replica node.
-
-  * Each node is running a 15 GB Rackspace Cloud Server.
-
-* Each app node has pgbouncer running on it pooling connections.
+* App nodes have pgbouncer running on it pooling connections.
 
   * The actual database user and password is only known to pgbouncer, each
     node will get a unique randomly generated password for the app to connect
     to pgbouncer.
 
-* The primary node also backs up to Rackspace CloudFiles in the ORD region
-  via WAL-E. A full backup is done once a week via a cronjob and WAL-E does
-  WAL pushes to fill in between the full backups.
 
+Local Tooling
+-------------
+
+For roles which require postgresql, the ``postgresql-primary`` vagrant machine
+can be booted to provide similar infrastructure to the DigitalOcean hosted
+Postgres.
 
 
 Creating a New Database
@@ -80,43 +77,3 @@ Giving Applications Access
               },
           },
       }
-
-
-Application Integration
------------------------
-
-The PostgreSQL has been configured to allow an application to integrate with it
-to get some advanced features.
-
-
-(A)synchronous Commit
-~~~~~~~~~~~~~~~~~~~~~
-
-By default the PostgreSQL primary will ensure that each transaction is commited
-to persistent storage on the local disk before returning that a transaction
-has successfully been commited. However it will asynchronously replicate that
-transaction to the replicas. This means that if the primary server goes down
-in a way where the disk is not recoverable prior to replication occuring than
-that data will be lost.
-
-Applications may optionally, on a per transaction basis, request that the
-primary server has either given the data to a replica server or that a replica
-server has also written that data to persistent storage.
-
-This can be acchived by executing:
-
-.. code-block:: plpgsql
-
-    -- Set the transaction so that a replica will have received the data, but
-    -- not written the data out before the primary says the transaction is
-    -- complete.
-    SET LOCAL synchronous_commit TO remote_write;
-
-    -- Set the transaction so that a replica will have written the data to
-    -- persistent storage before the primary says the transaction is complete.
-    SET LOCAL synchronous_commit TO on;
-
-Obviously each of these options will mean the write will fail if the primary
-cannot reach the replica server. These options can be used when ensuring data
-is saved is more important than uptime with the minimal risk the primary goes
-completely unrecoverable.

pillar/base/firewall/postgresql.sls

@@ -3,6 +3,3 @@
 firewall:
   postgresql:
     port: 5432
-
-fwmangle:
-  postgresql-stunnel: -A OUTPUT -p tcp -m multiport --sports 5431 -j MARK --set-xmark 0x1/0xffffffff

pillar/base/postgresql/server.sls

@@ -1,26 +1,12 @@
 postgresql:
-  databases:
-    codespeed-cpython:
-      owner: "codespeed-cpython"
-      cluster: "pg-nyc1-psf-io"
-    codespeed-pypy:
-      owner: "codespeed-pypy"
-      cluster: "pg-nyc1-psf-io"
-    roundup-cpython:
-      owner: "roundup-cpython"
-      cluster: pool-pg-nyc1-psf-io
-    roundup-jython:
-      owner: "roundup-jython"
-      cluster: pg-nyc1-psf-io
-    roundup-roundup:
-      owner: "roundup-roundup"
-      cluster: pg-nyc1-psf-io
-    roundup-cpython_test:
-      owner: "roundup-cpython_test"
-      cluster: pg-nyc1-psf-io
-    buildbot-master:
-      owner: "buildbot-master"
-      cluster: "pg-nyc1-psf-io"
-    buildbot-master_test:
-      owner: "buildbot-master_test"
-      cluster: "pg-nyc1-psf-io"
+  data_dir: /srv/postgresql/11/psf
+  config_dir: /etc/postgresql/11/psf
+  config_file: /etc/postgresql/11/psf/postgresql.conf
+  hba_file: /etc/postgresql/11/psf/pg_hba.conf
+  ident_file: /etc/postgresql/11/psf/pg_ident.conf
+  pid_file: /var/run/postgresql/11-psf.pid
+  recovery_file: /srv/postgresql/11/psf/recovery.conf
+
+  port: 5432
+  max_connections: 100
+  replicas: 1

pillar/base/tls.sls

@@ -6,6 +6,18 @@ tls:
     default: ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:ECDH+AES128:!aNULL:!SHA1:!AESCCM
 
   gen_certs:
+    bootstrap.pypa.psf.io:
+      roles:
+        - web-pypa
+
+    buildbot-master.psf.io:
+      roles:
+        - buildbot
+
+    codespeed.psf.io:
+      roles:
+        - codespeed
+
     consul.psf.io:
       roles:
         - consul
@@ -26,17 +38,13 @@ tls:
       roles:
         - loadbalancer
 
-    codespeed.psf.io:
-      roles:
-        - codespeed
-
-    bootstrap.pypa.psf.io:
+    moin.psf.io:
       roles:
-        - web-pypa
+        - moin
 
-    buildbot-master.psf.io:
+    postgresql.psf.io:
       roles:
-        - buildbot
+        - postgresql
 
     salt.psf.io:
       roles:
@@ -49,7 +57,3 @@ tls:
     pypy-web.psf.io:
       roles:
         - pypy-web
-
-    moin.psf.io:
-      roles:
-        - moin

pillar/dev/postgres/clusters.sls

@@ -0,0 +1,6 @@
+postgresql-clusters:
+  pg-vagrant-psf-io:
+    host: postgresql-primary.vagrant.psf.io
+    port: 5432
+    sslmode: verify-ca
+    ca_cert_pillar: 'tls:ca:PSF_CA'

pillar/dev/postgres/databases.sls

@@ -0,0 +1,25 @@
+postgresql-databases:
+  codespeed-cpython:
+    owner: "codespeed-cpython"
+    cluster: "pg-vagrant-psf-io"
+  codespeed-pypy:
+    owner: "codespeed-pypy"
+    cluster: "pg-vagrant-psf-io"
+  roundup-cpython:
+    owner: "roundup-cpython"
+    cluster: "pg-vagrant-psf-io"
+  roundup-jython:
+    owner: "roundup-jython"
+    cluster: "pg-vagrant-psf-io"
+  roundup-roundup:
+    owner: "roundup-roundup"
+    cluster: "pg-vagrant-psf-io"
+  roundup-cpython_test:
+    owner: "roundup-cpython_test"
+    cluster: "pg-vagrant-psf-io"
+  buildbot-master:
+    owner: "buildbot-master"
+    cluster: "pg-vagrant-psf-io"
+  buildbot-master_test:
+    owner: "buildbot-master_test"
+    cluster: "pg-vagrant-psf-io"

pillar/dev/roles.sls

@@ -50,6 +50,18 @@ roles:
     pattern: "mail.vagrant.psf.io"
     purpose: ""
     contact: ""
+  postgresql:
+    pattern: "postgresql-*.vagrant.psf.io"
+    purpose: ""
+    contact: ""
+  postgresql-primary:
+    pattern: "postgresql-primary.vagrant.psf.io"
+    purpose: ""
+    contact: ""
+  postgresql-replica:
+    pattern: "postgresql-replica.vagrant.psf.io"
+    purpose: ""
+    contact: ""
   planet:
     pattern: "planet.vagrant.psf.io"
     purpose: ""

pillar/dev/secrets/postgresql-admin.sls

@@ -0,0 +1,5 @@
+postgres-admin:
+  pg-vagrant-psf-io:
+    user: salt-master
+    password: insecurepasswordlol
+    database: defaultdb

pillar/dev/secrets/postgresql-users/all.sls

@@ -1,15 +1,39 @@
 postgresql-users:
-  bugs-python: insecurepasswordlol
-  monitoring: insecurepasswordlol
-  pydotorg-prod: insecurepasswordlol
-  pydotorg-staging: insecurepasswordlol
-  pycon-prod: insecurepasswordlol
-  pycon-staging: insecurepasswordlol
-  speed-web: insecurepasswordlol
-  testpypi: insecurepasswordlol
-  discourse-user: insecurepasswordlol
+  codespeed-cpython:
+    cluster: pg-vagrant-psf-io
+    dbname: codespeed-cpython
+    password: insecurepasswordlol
+  codespeed-pypy:
+    cluster: pg-vagrant-psf-io
+    dbname: codespeed-pypy
+    password: insecurepasswordlol
+  roundup-cpython:
+    cluster: pg-vagrant-psf-io
+    dbname: roundup-cpython
+    password: insecurepasswordlol
+  roundup-jython:
+    cluster: pg-vagrant-psf-io
+    dbname: roundup-jython
+    password: insecurepasswordlol
+  roundup-roundup:
+    cluster: pg-vagrant-psf-io
+    dbname: roundup-roundup
+    password: insecurepasswordlol
+  roundup-cpython_test:
+    cluster: pg-vagrant-psf-io
+    dbname: roundup-cpython_test
+    password: insecurepasswordlol
+  buildbot-master:
+    cluster: pg-vagrant-psf-io
+    dbname: buildbot-master
+    password: insecurepasswordlol
+  buildbot-master_test:
+    cluster: pg-vagrant-psf-io
+    dbname: buildbot-master_test
+    password: insecurepasswordlol
 
 postgresql-superusers:
-  diamond: insecurepasswordlol
+  salt-master:
+    password: insecurepasswordlol
 
 postgresql-replicator: insecurereplicatorpasswordlol

pillar/dev/top.sls

@@ -7,6 +7,7 @@ base:
     - sudoers
     - tls
     - users
+    - postgres.clusters
 
   'backup-server':
     - match: nodegroup
@@ -69,9 +70,11 @@ base:
     - match: nodegroup
     - firewall.postgresql
     - postgresql.server
+    - postgres.databases
 
   'postgresql-primary':
     - match: nodegroup
+    - secrets.postgresql-admin
     - secrets.postgresql-users.all
 
   'postgresql-replica':

pillar/prod/postgres/databases.sls

@@ -0,0 +1,25 @@
+postgresql-databases:
+  codespeed-cpython:
+    owner: "codespeed-cpython"
+    cluster: "pg-nyc1-psf-io"
+  codespeed-pypy:
+    owner: "codespeed-pypy"
+    cluster: "pg-nyc1-psf-io"
+  roundup-cpython:
+    owner: "roundup-cpython"
+    cluster: pg-nyc1-psf-io
+  roundup-jython:
+    owner: "roundup-jython"
+    cluster: pg-nyc1-psf-io
+  roundup-roundup:
+    owner: "roundup-roundup"
+    cluster: pg-nyc1-psf-io
+  roundup-cpython_test:
+    owner: "roundup-cpython_test"
+    cluster: pg-nyc1-psf-io
+  buildbot-master:
+    owner: "buildbot-master"
+    cluster: "pg-nyc1-psf-io"
+  buildbot-master_test:
+    owner: "buildbot-master_test"
+    cluster: "pg-nyc1-psf-io"

pillar/prod/top.sls

@@ -107,8 +107,7 @@ base:
   'salt-master':
     - match: nodegroup
     - firewall.salt
-    - secrets.dyn
+    - postgres.databases
     - secrets.aws
     - secrets.postgresql-admin
-    - postgresql.server
     - secrets.postgresql-users.all

salt/base/salt.sls

@@ -114,31 +114,3 @@ salt-minion:
     - restart: True
     - watch:
       - file: /etc/salt/minion.d/mine.conf
-
-{% if 'postgresql-admin' in pillar %}
-{% set postgresql = salt["pillar.get"]("postgresql", {}) %}
-{% for user, settings in salt["pillar.get"]("postgresql-users", {}).items() %}
-{{ user }}-user:
-  postgres_user.present:
-    - name: {{ user }}
-    - password: {{ settings['password'] }}
-    - refresh_password: True
-    - db_host: {{ pillar['postgresql-clusters'][settings['cluster']]['host'] }}
-    - db_port: {{ pillar['postgresql-clusters'][settings['cluster']]['port'] }}
-    - db_user: {{ pillar['postgresql-admin'][settings['cluster']]['user'] }}
-    - db_password: {{ pillar['postgresql-admin'][settings['cluster']]['password'] }}
-{% endfor %}
-
-{% for database, settings in postgresql.get("databases", {}).items() %}
-{{ database }}-database:
-  postgres_database.present:
-    - name: {{ database }}
-    - owner: {{ settings['owner'] }}
-    - db_host: {{ pillar['postgresql-clusters'][settings['cluster']]['host'] }}
-    - db_port: {{ pillar['postgresql-clusters'][settings['cluster']]['port'] }}
-    - db_user: {{ pillar['postgresql-admin'][settings['cluster']]['user'] }}
-    - db_password: {{ pillar['postgresql-admin'][settings['cluster']]['password'] }}
-    - require:
-      - postgres_user: {{ settings['owner'] }}-user
-{% endfor %}
-{% endif %}