diff --git a/README.md b/README.md index 4fdde2820..8d4cb2cba 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,16 @@ See all installation instructions in the [repo wiki](https://github.com/supabase [![Digital Ocean](https://github.com/supabase/postgres/blob/master/docs/img/digital-ocean.png)](https://github.com/supabase/postgres/wiki/Digital-Ocean) [![AWS](https://github.com/supabase/postgres/blob/master/docs/img/aws.png)](https://github.com/supabase/postgres/wiki/AWS-EC2) +### Building + +Set the `supabase_internal` flag to `false` to avoid baking in components that are specific to Supabase's hosted offering. + +```bash +$ time packer build -timestamp-ui \ + -var "ansible_arguments=--skip-tags,update-only,-v,-e,supabase_internal='false'" \ + amazon-arm.json +``` + ## Motivation After talking to a lot of techies, we've found that most believe Postgres is the best (operational) database but they _still_ choose other databases. This is overwhelmingly because "the other one was quicker/easier". Our goal is to make it fast and simple to get started with Postgres, so that we never hear that excuse again. diff --git a/ansible/files/exporter.service.j2 b/ansible/files/exporter.service.j2 new file mode 100644 index 000000000..cbf56d3c9 --- /dev/null +++ b/ansible/files/exporter.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Postgres Exporter + +[Service] +Type=simple +ExecStart=/opt/postgres_exporter/postgres_exporter --auto-discover-databases --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics +User=root +StandardOutput=file:/var/log/postgres_exporter.stdout +StandardError=file:/var/log/postgres_exporter.error +Restart=always +RestartSec=3 +Environment="DATA_SOURCE_URI=localhost/postgres" + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/pgbouncer_auth_schema.sql b/ansible/files/pgbouncer_auth_schema.sql index 9807ccd47..bc1342f8d 100644 --- a/ansible/files/pgbouncer_auth_schema.sql +++ b/ansible/files/pgbouncer_auth_schema.sql @@ -1,13 +1,13 @@ CREATE USER pgbouncer; CREATE SCHEMA pgbouncer AUTHORIZATION pgbouncer; - + CREATE OR REPLACE FUNCTION pgbouncer.get_auth(p_usename TEXT) RETURNS TABLE(username TEXT, password TEXT) AS $$ BEGIN RAISE WARNING 'PgBouncer auth request: %', p_usename; - + RETURN QUERY SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow WHERE usename = p_usename; @@ -15,4 +15,4 @@ END; $$ LANGUAGE plpgsql SECURITY DEFINER; REVOKE ALL ON FUNCTION pgbouncer.get_auth(p_usename TEXT) FROM PUBLIC; -GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO pgbouncer; \ No newline at end of file +GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO pgbouncer; diff --git a/ansible/files/queries.yml.j2 b/ansible/files/queries.yml.j2 new file mode 100644 index 000000000..7f1098b5c --- /dev/null +++ b/ansible/files/queries.yml.j2 @@ -0,0 +1,45 @@ +pg_database: + query: "SELECT SUM(pg_database_size(pg_database.datname)) / (1024 * 1024) as size_mb FROM pg_database" + master: true + cache_seconds: 30 + metrics: + - size_mb: + usage: "GAUGE" + description: "Disk space used by the database" + +pg_stat_database: + query: "SELECT sum(numbackends) as num_backends FROM pg_stat_database" + master: true + metrics: + - num_backends: + usage: "GAUGE" + description: "The number of active backends" + +pg_stat_statements: + query: "SELECT sum(calls) as total_queries, sum(total_time / 1000) as total_time_seconds FROM pg_stat_statements t1 JOIN pg_database t3 ON (t1.dbid=t3.oid)" + master: true + metrics: + - total_queries: + usage: "COUNTER" + description: "Number of times executed" + - total_time_seconds: + usage: "COUNTER" + description: "Total time spent, in seconds" + +auth_users: + query: "select count(id) as user_count from auth.users" + master: true + cache_seconds: 30 + metrics: + - user_count: + usage: "GAUGE" + description: "Number of users in the project db" + +storage: + query: "select sum(size) / (1024 * 1024) as storage_size_mb from storage.get_size_by_bucket()" + master: true + cache_seconds: 30 + metrics: + - storage_size_mb: + usage: "GAUGE" + description: "The total size used for all storage buckets, in mb" diff --git a/ansible/files/stat_extension.sql b/ansible/files/stat_extension.sql new file mode 100644 index 000000000..841ff0c4b --- /dev/null +++ b/ansible/files/stat_extension.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; diff --git a/ansible/playbook.yml b/ansible/playbook.yml index ab94c525a..845e51c3f 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -7,6 +7,11 @@ vars_files: - ./vars.yml + vars: + sql_files: + - { source: "pgbouncer_auth_schema.sql", dest: "00-schema.sql" } + - { source: "stat_extension.sql", dest: "01-extension.sql" } + roles: - role: anxs.postgresql @@ -17,25 +22,32 @@ - name: Install Postgres extensions import_tasks: tasks/setup-extensions.yml + - name: Install Supabase specific content + import_tasks: tasks/setup-supabase-internal.yml + when: supabase_internal == true + - name: Adjust APT update intervals copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - - name: Dump PgBouncer auth schema + - name: Transfer init SQL files copy: - src: files/pgbouncer_auth_schema.sql - dest: /tmp/00-schema.sql + src: files/{{ item.source }} + dest: /tmp/{{ item.dest }} + loop: "{{ sql_files }}" - - name: Set up readonly user for the public schema + - name: Execute init SQL files become_user: postgres - shell: - cmd: psql -f /tmp/00-schema.sql + shell: + cmd: psql -f /tmp/{{ item.dest }} + loop: "{{ sql_files }}" - - name: Delete SQL script + - name: Delete SQL scripts file: - path: /tmp/00-schema.sql + path: /tmp/{{ item.dest }} state: absent + loop: "{{ sql_files }}" - name: Adjust pgbouncer.ini copy: @@ -57,7 +69,7 @@ rule: allow port: "6543" - - name: UFW - Deny all other incoming traffix by default + - name: UFW - Deny all other incoming traffic by default ufw: state: enabled policy: deny diff --git a/ansible/tasks/setup-misc.yml b/ansible/tasks/setup-misc.yml index c7136827d..756d66227 100644 --- a/ansible/tasks/setup-misc.yml +++ b/ansible/tasks/setup-misc.yml @@ -54,4 +54,4 @@ become: yes apt: pkg: - - pgbouncer \ No newline at end of file + - pgbouncer diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml new file mode 100644 index 000000000..1304b810c --- /dev/null +++ b/ansible/tasks/setup-supabase-internal.yml @@ -0,0 +1,65 @@ +- name: AWS CLI dep + apt: + pkg: + - unzip + install_recommends: no + +- name: AWS CLI + get_url: + url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{{ aws_cli_release }}.zip" + dest: "/tmp/awscliv2.zip" + +- name: AWS CLI - expand + unarchive: + remote_src: yes + src: "/tmp/awscliv2.zip" + dest: "/tmp" + +- name: AWS CLI - install + shell: "/tmp/aws/install" + become: true + +- name: UFW - Allow connections to exporter for prometheus + ufw: + rule: allow + port: "9187" + +- name: create directories + file: + state: directory + path: "{{ item }}" + owner: root + mode: '0700' + become: yes + with_items: + - /opt/postgres_exporter + - /etc/systemd/system/exporter.service.d + +- name: download postgres exporter + get_url: + url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ postgres_exporter_release }}/postgres_exporter-{{ postgres_exporter_release }}.linux-arm64.tar.gz" + dest: /tmp/postgres_exporter.tar.gz + +- name: expand postgres exporter + unarchive: + remote_src: yes + src: /tmp/postgres_exporter.tar.gz + dest: /opt/postgres_exporter + extra_opts: [--strip-components=1] + become: yes + +- name: exporter create a service + template: + src: files/exporter.service.j2 + dest: /etc/systemd/system/postgres_exporter.service + +- name: exporter copy over queries + template: + src: files/queries.yml.j2 + dest: /opt/postgres_exporter/queries.yml + +- name: exporter enable service + systemd: + enabled: yes + name: postgres_exporter + daemon_reload: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 9ab9f8021..78532148b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,3 +1,5 @@ +supabase_internal: true + postgresql_version: 12 postgresql_wal_level: "logical" postgresql_max_wal_senders: 10 @@ -92,3 +94,8 @@ postgresql_log_rotation_size: 0 golang_version: "1.15.4" wal_g_release: "0.2.15" wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 + +postgres_exporter_release: "0.9.0" +postgres_exporter_release_checksum: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 + +aws_cli_release: "2.0.30"