From 87401749dfbeb0b6800a402cde513f0cb904fed5 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:10:56 +0200 Subject: [PATCH 01/27] chore(git): Add Gitignore --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba429be --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.vscode +/cover.html +/coverage.txt +/coverage.xml +/dist +/postgresql-partition-manager +/postgresql-partition-manager.yaml +scripts/localdev/.data From 75a096e462bcb5820ebc6463b317cd7548dc5309 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:29:04 +0200 Subject: [PATCH 02/27] chore(github): Add Github issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 36 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 24 +++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..21e6513 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,36 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +### Describe the bug + +A clear and concise description of what the bug is. + +### Desktop (please complete the following information) + +- OS: [e.g. Linux] +- PostgreSQL partition manager [e.g. v1.1] +- PostgreSQL version: [e.g. v14] +- Partition key column type [e.g. date, uuidv7] + +If possible, please also share: + +- Table table structure dump (`pg_dump --schema-only --no-comments --no-privileges -t `) +- PostgreSQL Partition Manager configuration (`postgresql-partition-manager.yaml`) + +### To Reproduce + +Steps to reproduce the behavior + +### Expected behavior + +A clear and concise description of what you expected to happen. + +### Additional context + +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..dd40f22 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,24 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +### Is your feature request related to a problem? Please describe + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +### Describe the solution you'd like + +A clear and concise description of what you want to happen. + +### Describe alternatives you've considered + +A clear and concise description of any alternative solutions or features you've considered. + +### Additional context + +Add any other context or screenshots about the feature request here. From 535a4755c47285917857d6a02b863f1d946584ad Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:29:27 +0200 Subject: [PATCH 03/27] chore(github): Configure dependabot --- .github/dependabot.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..7ba20f7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +--- +version: 2 +updates: + + - package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 10 + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: weekly + + - package-ecosystem: "docker" + directory: "/scripts/localdev" + schedule: + interval: weekly From 637e383d0ed902412bc4b499fe020392764cba92 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:12:53 +0200 Subject: [PATCH 04/27] chore(linter): Add linters --- .golangci.yml | 94 +++++++++++++++++++++++++++++++++++++++++ .markdownlint.yaml | 8 ++++ .pre-commit-config.yaml | 28 ++++++++++++ .yamllint.yaml | 11 +++++ 4 files changed, 141 insertions(+) create mode 100644 .golangci.yml create mode 100644 .markdownlint.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint.yaml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..5ed6719 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,94 @@ +--- +run: + concurrency: 4 + deadline: 2m + issues-exit-code: 1 + tests: true + +output: + format: colored-line-number + print-issued-lines: true + print-linter-name: true + +linters: + enable-all: false + disable-all: false + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - asciicheck + - bodyclose + - dogsled + - durationcheck + - errorlint + - exhaustive + - exportloopref + - forcetypeassert + - gochecknoinits + - goconst + - gocritic + - gocyclo + - godox + - goerr113 + - gofmt + - gofumpt + - goimports + - gomnd + - gomodguard + - goprintffuncname + - gosec + - importas + - makezero + - misspell + - nakedret + - nestif + - nilerr + - nlreturn + - noctx + - nolintlint + - prealloc + - predeclared + - revive + - rowserrcheck + - sqlclosecheck + - stylecheck + - testpackage + - thelper + - tparallel + - unconvert + - unparam + - wastedassign + - whitespace + - wrapcheck + - wsl + +linters-settings: + gocyclo: + min-complexity: 35 + + revive: + rules: + - name: exported + disabled: true + + lll: + line-length: 120 + +issues: + exclude-use-default: false + max-per-linter: 1024 + max-same: 1024 + + exclude-rules: + # Exclude some linters from running on test files + - path: _test\.go + linters: + # bodyclose reports some false-positives when using a test request recorder + - bodyclose + # It's overkill to use `NewRequestWithContext` in tests + - noctx diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..948176b --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,8 @@ +--- +# https://github.com/DavidAnson/markdownlint + +# MD013/line-length - Line length +MD013: false + +# MD033/no-inline-html - Inline HTML +MD033: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2d3b91d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: check-yaml + exclude: configs/helm/templates/ + - id: check-json + - id: end-of-file-fixer + - id: detect-private-key + - id: check-symlinks + - repo: https://github.com/golangci/golangci-lint + rev: v1.54.2 + hooks: + - id: golangci-lint + - repo: https://github.com/Bahjat/pre-commit-golang + rev: v1.0.3 + hooks: + - id: gofumpt # requires https://github.com/mvdan/gofumpt + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.37.0 + hooks: + - id: markdownlint # requires https://github.com/DavidAnson/markdownlint-cli2 + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.32.0 + hooks: + - id: yamllint # requires https://github.com/adrienverge/yamllint diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..dabaed0 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,11 @@ +--- +extends: default + +rules: + line-length: + max: 200 + level: warning + +ignore: + - dist/ + - configs/helm/templates/*.yaml # Helm templates are invalid due to Go template language From 5460841cdfbb70631967b31aa31f14a3b0744932 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:15:14 +0200 Subject: [PATCH 05/27] chore(localdev): Add local development environment --- .../localdev/configuration/bats/Dockerfile | 3 ++ .../postgres/seeds/00_database.sql | 1 + .../postgres/seeds/10_by_date.sql | 15 ++++++++ .../postgres/seeds/10_by_timestamp.sql | 15 ++++++++ .../postgres/seeds/10_by_uuidv7.sql | 24 ++++++++++++ scripts/localdev/docker-compose.yml | 37 +++++++++++++++++++ 6 files changed, 95 insertions(+) create mode 100644 scripts/localdev/configuration/bats/Dockerfile create mode 100644 scripts/localdev/configuration/postgres/seeds/00_database.sql create mode 100644 scripts/localdev/configuration/postgres/seeds/10_by_date.sql create mode 100644 scripts/localdev/configuration/postgres/seeds/10_by_timestamp.sql create mode 100644 scripts/localdev/configuration/postgres/seeds/10_by_uuidv7.sql create mode 100644 scripts/localdev/docker-compose.yml diff --git a/scripts/localdev/configuration/bats/Dockerfile b/scripts/localdev/configuration/bats/Dockerfile new file mode 100644 index 0000000..e27800c --- /dev/null +++ b/scripts/localdev/configuration/bats/Dockerfile @@ -0,0 +1,3 @@ +FROM bats/bats:1.11.0 + +RUN apk add postgresql-client yq diff --git a/scripts/localdev/configuration/postgres/seeds/00_database.sql b/scripts/localdev/configuration/postgres/seeds/00_database.sql new file mode 100644 index 0000000..b1a3a91 --- /dev/null +++ b/scripts/localdev/configuration/postgres/seeds/00_database.sql @@ -0,0 +1 @@ +CREATE DATABASE partitions; diff --git a/scripts/localdev/configuration/postgres/seeds/10_by_date.sql b/scripts/localdev/configuration/postgres/seeds/10_by_date.sql new file mode 100644 index 0000000..f81bade --- /dev/null +++ b/scripts/localdev/configuration/postgres/seeds/10_by_date.sql @@ -0,0 +1,15 @@ +\c partitions; + +CREATE TABLE by_date ( + id BIGSERIAL, + temperature INT, + created_at DATE NOT NULL +) PARTITION BY RANGE (created_at); + +CREATE TABLE by_date_2000 PARTITION OF by_date FOR VALUES FROM ('2000-01-01') TO ('2001-01-01'); +CREATE TABLE by_date_2001 PARTITION OF by_date FOR VALUES FROM ('2001-01-01') TO ('2002-01-01'); +CREATE TABLE by_date_2002 PARTITION OF by_date FOR VALUES FROM ('2002-01-01') TO ('2003-01-01'); + +INSERT INTO by_date values (1, floor(RANDOM()*100), '2000-12-31'); +INSERT INTO by_date values (2, floor(RANDOM()*100), '2001-12-31'); +INSERT INTO by_date values (3, floor(RANDOM()*100), '2002-12-31'); diff --git a/scripts/localdev/configuration/postgres/seeds/10_by_timestamp.sql b/scripts/localdev/configuration/postgres/seeds/10_by_timestamp.sql new file mode 100644 index 0000000..364f075 --- /dev/null +++ b/scripts/localdev/configuration/postgres/seeds/10_by_timestamp.sql @@ -0,0 +1,15 @@ +\c partitions; + +CREATE TABLE by_timestamp ( + id BIGSERIAL, + temperature INT, + created_at TIMESTAMP NOT NULL +) PARTITION BY RANGE (created_at); + +CREATE TABLE by_timestamp_2000 PARTITION OF by_timestamp FOR VALUES FROM ('2000-01-01') TO ('2001-01-01'); +CREATE TABLE by_timestamp_2001 PARTITION OF by_timestamp FOR VALUES FROM ('2001-01-01') TO ('2002-01-01'); +CREATE TABLE by_timestamp_2002 PARTITION OF by_timestamp FOR VALUES FROM ('2002-01-01') TO ('2003-01-01'); + +INSERT INTO by_timestamp values (1, floor(RANDOM()*100), '2000-12-31T23:54:00'); +INSERT INTO by_timestamp values (2, floor(RANDOM()*100), '2001-12-31T23:54:00'); +INSERT INTO by_timestamp values (3, floor(RANDOM()*100), '2002-12-31T23:54:00'); diff --git a/scripts/localdev/configuration/postgres/seeds/10_by_uuidv7.sql b/scripts/localdev/configuration/postgres/seeds/10_by_uuidv7.sql new file mode 100644 index 0000000..f7d92c2 --- /dev/null +++ b/scripts/localdev/configuration/postgres/seeds/10_by_uuidv7.sql @@ -0,0 +1,24 @@ +\c partitions; + +CREATE OR REPLACE FUNCTION min_uuid_v7(min_timestamp timestamp) RETURNS uuid +LANGUAGE sql immutable strict +AS $function$ + SELECT encode( + overlay('\x00000000000070000000000000000000'::bytea + placing substring(int8send(floor(extract(epoch from min_timestamp) * 1000)::bigint) from 3) + from 1 for 6) + , 'hex')::uuid; +$function$; + +CREATE TABLE by_uuidv7 ( + id UUID, + temperature INT +) PARTITION BY RANGE (id); + +CREATE TABLE by_uuidv7_2000 PARTITION OF by_uuidv7 FOR VALUES FROM (min_uuid_v7('2000-01-01')) TO (min_uuid_v7('2001-01-01')); +CREATE TABLE by_uuidv7_2001 PARTITION OF by_uuidv7 FOR VALUES FROM (min_uuid_v7('2001-01-01')) TO (min_uuid_v7('2002-01-01')); +CREATE TABLE by_uuidv7_2002 PARTITION OF by_uuidv7 FOR VALUES FROM (min_uuid_v7('2002-01-01')) TO (min_uuid_v7('2003-01-01')); + +INSERT INTO by_uuidv7 values (min_uuid_v7('2000-01-01'), floor(RANDOM()*100)); +INSERT INTO by_uuidv7 values (min_uuid_v7('2001-01-01'), floor(RANDOM()*100)); +INSERT INTO by_uuidv7 values (min_uuid_v7('2002-01-01'), floor(RANDOM()*100)); diff --git a/scripts/localdev/docker-compose.yml b/scripts/localdev/docker-compose.yml new file mode 100644 index 0000000..0ed97f5 --- /dev/null +++ b/scripts/localdev/docker-compose.yml @@ -0,0 +1,37 @@ +--- +version: '3.1' + +volumes: + postgresql_data: {} + +services: + + postgres: + image: postgres:${POSTGRESQL_VERSION:-16} + environment: + POSTGRES_PASSWORD: hackme + read_only: true + security_opt: + - no-new-privileges:true + ports: + - 5432:5432 + volumes: + - ./.data/postgres:/var/lib/postgresql/data + - ./configuration/postgres/seeds:/docker-entrypoint-initdb.d + command: "postgres -c log_statement=all -c log_line_prefix='%t:%r:user=%u,database=%d,app=%a,query_id=%Q:[%p]:'" + tmpfs: + - /var/run + + bats: + build: + context: configuration/bats + environment: + PGHOST: postgresql + PGUSER: postgres + PGPASSWORD: hackme + PGDATABASE: unittest + command: + - /code + volumes: + - ../../postgresql-partition-manager:/usr/local/bin/postgresql-partition-manager + - ../bats:/code From ef7817e84b0ea85c4aa1884816be6066ea0caf80 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:14:28 +0200 Subject: [PATCH 06/27] chore(go): Add initial project structure --- Makefile | 41 ++++++++++ go.mod | 56 +++++++++++++ go.sum | 136 ++++++++++++++++++++++++++++++++ internal/infra/build/build.go | 8 ++ internal/infra/logger/logger.go | 25 ++++++ 5 files changed, 266 insertions(+) create mode 100644 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/infra/build/build.go create mode 100644 internal/infra/logger/logger.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2ad23fa --- /dev/null +++ b/Makefile @@ -0,0 +1,41 @@ +SHELL=/bin/bash -o pipefail +BUILD_INFO_PACKAGE_PATH=github.com/qonto/postgresql-partition-manager/internal/infra/build +BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') +GIT_COMMIT_SHA=$(shell git rev-parse HEAD) +BINARY=postgresql-partition-manager +ARCHITECTURE=$(shell uname -m) + +all: build + +.PHONY: format +format: + gofumpt -l -w . + +.PHONY: build +build: + CGO_ENABLED=0 go build -v -ldflags="-X '$(BUILD_INFO_PACKAGE_PATH).Version=development' -X '$(BUILD_INFO_PACKAGE_PATH).CommitSHA=$(GIT_COMMIT_SHA)' -X '$(BUILD_INFO_PACKAGE_PATH).Date=$(BUILD_DATE)'" -o $(BINARY) + +.PHONY: run +run: + ./$(BINARY) $(args) + +.PHONY: install +install: build + GOBIN=/usr/local/bin/ go install -v -ldflags="-X '$(BUILD_INFO_PACKAGE_PATH).Version=development' -X '$(BUILD_INFO_PACKAGE_PATH).CommitSHA=$(GIT_COMMIT_SHA)' -X '$(BUILD_INFO_PACKAGE_PATH).Date=$(BUILD_DATE)'" + +.PHONY: bats-test +bats-test: + cd scripts/bats && bats *.bats + +.PHONY: test +test: + go test -race -v ./... -coverprofile=coverage.txt -covermode atomic + go run github.com/boumenot/gocover-cobertura@latest < coverage.txt > coverage.xml + go tool cover -html coverage.txt -o cover.html + +.PHONY: lint +lint: + golangci-lint run --verbose --timeout 2m + +.PHONY: all-tests +all-tests: test goreleaser-check diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a33b351 --- /dev/null +++ b/go.mod @@ -0,0 +1,56 @@ +module github.com/qonto/postgresql-partition-manager + +go 1.21.3 + +toolchain go1.21.9 + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.14.3 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pashagolub/pgxmock/v3 v3.3.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/samborkent/uuidv7 v0.0.0-20231110121620-f2e19d87e48b // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.18.2 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.9.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.20.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools v2.2.0+incompatible // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d15db62 --- /dev/null +++ b/go.sum @@ -0,0 +1,136 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pashagolub/pgxmock/v3 v3.3.0 h1:vMDQiBs74JEIYT/DeWNtUDrcfKCsgMmKd+ecQs1WsV4= +github.com/pashagolub/pgxmock/v3 v3.3.0/go.mod h1:ywwoE43oyD7aqpA3Jh5tvZ8h00P7RRiygA23aXmNpWU= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= +github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/samborkent/uuidv7 v0.0.0-20231110121620-f2e19d87e48b h1:39v+thWy220bPAl5iP0p0b1s5DXmrtidMFRZqYsmEfI= +github.com/samborkent/uuidv7 v0.0.0-20231110121620-f2e19d87e48b/go.mod h1:Z46aLAe76cDDo+W1m5zVg+KeB+4P2+xWENVEFFzbBuQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/internal/infra/build/build.go b/internal/infra/build/build.go new file mode 100644 index 0000000..ce0aae1 --- /dev/null +++ b/internal/infra/build/build.go @@ -0,0 +1,8 @@ +// Package build store build information (version, date, ...) +package build + +var ( + Version = "unknown" + CommitSHA = "unknown" + Date = "unknown" +) diff --git a/internal/infra/logger/logger.go b/internal/infra/logger/logger.go new file mode 100644 index 0000000..84704c3 --- /dev/null +++ b/internal/infra/logger/logger.go @@ -0,0 +1,25 @@ +// Package logger implements logging methods +package logger + +import ( + "log/slog" + "os" +) + +func New(debug bool, logFormat string) (*slog.Logger, error) { + logLevel := &slog.LevelVar{} + if debug { + logLevel.Set(slog.LevelDebug) + } + + opts := slog.HandlerOptions{ + Level: logLevel, + } + + logger := slog.New(slog.NewTextHandler(os.Stdout, &opts)) + if logFormat == "json" { + logger = slog.New(slog.NewJSONHandler(os.Stdout, &opts)) + } + + return logger, nil +} From e1733c209bbe0f27a156930dbcb93d96432a0478 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:15:49 +0200 Subject: [PATCH 07/27] chore(go): Add retry package --- internal/infra/retry/retry.go | 20 +++++++++++ internal/infra/retry/retry_test.go | 53 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 internal/infra/retry/retry.go create mode 100644 internal/infra/retry/retry_test.go diff --git a/internal/infra/retry/retry.go b/internal/infra/retry/retry.go new file mode 100644 index 0000000..8ca330f --- /dev/null +++ b/internal/infra/retry/retry.go @@ -0,0 +1,20 @@ +// Package retry provides methods to retry operations +package retry + +import ( + "time" +) + +func WithRetry(maxRetries int, operation func(attempt int) error) error { + var err error + + for attempt := 1; attempt <= maxRetries; attempt++ { + if err = operation(attempt); err == nil { + return nil + } + + time.Sleep(time.Duration(attempt) * time.Second) + } + + return err +} diff --git a/internal/infra/retry/retry_test.go b/internal/infra/retry/retry_test.go new file mode 100644 index 0000000..ccedf66 --- /dev/null +++ b/internal/infra/retry/retry_test.go @@ -0,0 +1,53 @@ +package retry_test + +import ( + "errors" + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/retry" +) + +var ErrGeneric = errors.New("failed") + +func TestWithRetry(t *testing.T) { + t.Run("operation succeeds after 1 retry", func(t *testing.T) { + attempts := 0 + operation := func(_ int) error { + attempts++ + if attempts < 2 { + return ErrGeneric + } + + return nil + } + + err := retry.WithRetry(2, operation) + if err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + + if attempts != 2 { + t.Fatalf("expected 2 attempts, but got: %d", attempts) + } + }) + + t.Run("operation never succeeds", func(t *testing.T) { + maxRetries := 2 + attempts := 0 + operation := func(_ int) error { + attempts++ + + return ErrGeneric + } + + err := retry.WithRetry(maxRetries, operation) + + if err == nil { + t.Fatalf("expected error, but got none") + } + + if attempts != maxRetries { + t.Fatalf("expected %d attempts, but got: %d", maxRetries, attempts) + } + }) +} From 0888ed70ec3f104ed312407f034842e6f1eac796 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:16:22 +0200 Subject: [PATCH 08/27] chore(go): Add UUID7 package --- internal/infra/uuid7/uuid7.go | 38 +++++++++++++++++++++++++++++ internal/infra/uuid7/uuid7_test.go | 39 ++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 internal/infra/uuid7/uuid7.go create mode 100644 internal/infra/uuid7/uuid7_test.go diff --git a/internal/infra/uuid7/uuid7.go b/internal/infra/uuid7/uuid7.go new file mode 100644 index 0000000..2cff57d --- /dev/null +++ b/internal/infra/uuid7/uuid7.go @@ -0,0 +1,38 @@ +// Package uuid7 provides function to generate UUIDv7 from time +package uuid7 + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "time" +) + +//nolint:gomnd +func FromTime(timestamp time.Time) string { + // Convert timestamp to Unix time in milliseconds + unixMillis := timestamp.UnixNano() / int64(time.Millisecond) + + // Create a byte slice from the Unix time (48 bits, big endian) + // Ensure the slice is initially 8 bytes to accommodate the full uint64, + // but we'll only use the last 6 bytes for the timestamp + timeBytes := make([]byte, 8) + binary.BigEndian.PutUint64(timeBytes, uint64(unixMillis)) + timeBytes = timeBytes[2:] // Keep the last 6 bytes + + // Generate random bytes for the rest of the UUID (10 bytes to make it a total of 16) + randomBytes := make([]byte, 10) + + _, err := rand.Read(randomBytes) + if err != nil { + panic("Failed to generate random bytes") + } + + // Combine time bytes and random bytes + uuidBytes := append(timeBytes, randomBytes...) //nolint:gocritic,makezero + + // Encode the UUID bytes to a string + uuid := fmt.Sprintf("%x-%x-7000-0000-000000000000", uuidBytes[0:4], uuidBytes[4:6]) + + return uuid +} diff --git a/internal/infra/uuid7/uuid7_test.go b/internal/infra/uuid7/uuid7_test.go new file mode 100644 index 0000000..ecf036f --- /dev/null +++ b/internal/infra/uuid7/uuid7_test.go @@ -0,0 +1,39 @@ +package uuid7_test + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/qonto/postgresql-partition-manager/internal/infra/uuid7" + "github.com/stretchr/testify/assert" +) + +const UUIDv7Version uuid.Version = 7 + +func TestFromTime(t *testing.T) { + testCases := []struct { + timestamp time.Time + expected string + }{ + { + time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC), + "018d242a-c800-7000-0000-000000000000", + }, + } + + for _, tc := range testCases { + t.Run(tc.timestamp.String(), func(t *testing.T) { + generated := uuid7.FromTime(tc.timestamp) + + assert.Equal(t, generated, tc.expected, "Should match expected ") + + decoded, err := uuid.Parse(generated) + timestamp, _ := decoded.Time().UnixTime() + + assert.Nil(t, err, "UUID should be parsable") + assert.Equal(t, decoded.Version(), UUIDv7Version, "Should be an UUIDv7") + assert.Equal(t, timestamp, tc.timestamp.Unix(), "Timestamp from generated UUID should match") + }) + } +} From 12b426907cf7a6cbbff152f41b1c872ef950765e Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:17:34 +0200 Subject: [PATCH 09/27] chore(go): Add postgresql package --- internal/infra/postgresql/bounds.go | 141 +++++ internal/infra/postgresql/column.go | 60 +++ .../infra/postgresql/column_internal_test.go | 66 +++ internal/infra/postgresql/column_test.go | 32 ++ internal/infra/postgresql/connection.go | 33 ++ internal/infra/postgresql/error.go | 17 + .../infra/postgresql/error_internal_test.go | 45 ++ internal/infra/postgresql/partition.go | 35 ++ .../postgresql/partition_internal_test.go | 144 ++++++ internal/infra/postgresql/partition_test.go | 483 ++++++++++++++++++ .../postgresql/partitionconfiguration.go | 150 ++++++ .../infra/postgresql/partitionsettings.go | 23 + .../postgresql/partitionsettings_test.go | 63 +++ .../infra/postgresql/partitionstrategy.go | 9 + internal/infra/postgresql/postgresql.go | 366 +++++++++++++ .../postgresql/postgresql_internal_test.go | 272 ++++++++++ internal/infra/postgresql/server.go | 39 ++ internal/infra/postgresql/server_test.go | 36 ++ internal/infra/postgresql/table.go | 18 + internal/infra/postgresql/table_test.go | 32 ++ 20 files changed, 2064 insertions(+) create mode 100644 internal/infra/postgresql/bounds.go create mode 100644 internal/infra/postgresql/column.go create mode 100644 internal/infra/postgresql/column_internal_test.go create mode 100644 internal/infra/postgresql/column_test.go create mode 100644 internal/infra/postgresql/connection.go create mode 100644 internal/infra/postgresql/error.go create mode 100644 internal/infra/postgresql/error_internal_test.go create mode 100644 internal/infra/postgresql/partition.go create mode 100644 internal/infra/postgresql/partition_internal_test.go create mode 100644 internal/infra/postgresql/partition_test.go create mode 100644 internal/infra/postgresql/partitionconfiguration.go create mode 100644 internal/infra/postgresql/partitionsettings.go create mode 100644 internal/infra/postgresql/partitionsettings_test.go create mode 100644 internal/infra/postgresql/partitionstrategy.go create mode 100644 internal/infra/postgresql/postgresql.go create mode 100644 internal/infra/postgresql/postgresql_internal_test.go create mode 100644 internal/infra/postgresql/server.go create mode 100644 internal/infra/postgresql/server_test.go create mode 100644 internal/infra/postgresql/table.go create mode 100644 internal/infra/postgresql/table_test.go diff --git a/internal/infra/postgresql/bounds.go b/internal/infra/postgresql/bounds.go new file mode 100644 index 0000000..859abfa --- /dev/null +++ b/internal/infra/postgresql/bounds.go @@ -0,0 +1,141 @@ +package postgresql + +import ( + "errors" + "fmt" + "time" + + "github.com/google/uuid" +) + +const ( + UUIDv7Version uuid.Version = 7 +) + +var ( + ErrLowerBoundAfterUpperBound = errors.New("lowerbound is after upperbound") + + // ErrCantDecodePartitionBounds represents an error indicating that the partition bounds cannot be decoded. + ErrCantDecodePartitionBounds = errors.New("partition bounds cannot be decoded") + + ErrUnsupportedUUIDVersion = errors.New("unsupported UUID version") +) + +func getDailyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, date.UTC().Location()) + upperBound = lowerBound.AddDate(0, 0, 1) + + return +} + +func getWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, date.UTC().Location()).AddDate(0, 0, -int(date.Weekday()-time.Monday)) + upperBound = lowerBound.AddDate(0, 0, daysInAweek) + + return +} + +func getMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, date.UTC().Location()) + upperBound = lowerBound.AddDate(0, 1, 0) + + return +} + +func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, date.UTC().Location()) + upperBound = lowerBound.AddDate(1, 0, 0) + + return +} + +func parseBounds(partition Partition) (lowerBound time.Time, upperBound time.Time, err error) { + lowerBound, upperBound, err = parseBoundAsTime(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsDate(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsDateTime(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsUUIDv7(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + if lowerBound.After(lowerBound) { + return time.Time{}, time.Time{}, ErrLowerBoundAfterUpperBound + } + + return time.Time{}, time.Time{}, ErrCantDecodePartitionBounds +} + +func parseBoundAsTime(partition Partition) (lowerBound, upperBound time.Time, err error) { + lowerBound, ok := partition.LowerBound.(time.Time) + if !ok { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as time: %w", err) + } + + upperBound, ok = partition.UpperBound.(time.Time) + if !ok { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as time: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsDate(partition Partition) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02", partition.LowerBound.(string)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as date: %w", err) + } + + upperBound, err = time.Parse("2006-01-02", partition.UpperBound.(string)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as date: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsDateTime(partition Partition) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02 15:04:05", partition.LowerBound.(string)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime: %w", err) + } + + upperBound, err = time.Parse("2006-01-02 15:04:05", partition.UpperBound.(string)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsUUIDv7(partition Partition) (lowerBound, upperBound time.Time, err error) { + lowerBoundUUID, err := uuid.Parse(partition.LowerBound.(string)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as UUID: %w", err) + } + + upperBoundUUID, err := uuid.Parse(partition.UpperBound.(string)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as UUID: %w", err) + } + + if upperBoundUUID.Version() != UUIDv7Version || lowerBoundUUID.Version() != UUIDv7Version { + return time.Time{}, time.Time{}, ErrUnsupportedUUIDVersion + } + + upperBound = time.Unix(upperBoundUUID.Time().UnixTime()).UTC() + lowerBound = time.Unix(lowerBoundUUID.Time().UnixTime()).UTC() + + return lowerBound, upperBound, nil +} diff --git a/internal/infra/postgresql/column.go b/internal/infra/postgresql/column.go new file mode 100644 index 0000000..5aed69a --- /dev/null +++ b/internal/infra/postgresql/column.go @@ -0,0 +1,60 @@ +package postgresql + +import ( + "errors" + "fmt" + "strings" +) + +const ( + DateColumnType ColumnType = "date" + DateTimeColumnType ColumnType = "timestamp" + UUIDColumnType ColumnType = "uuid" +) + +// ErrUnsupportedPartitionKeyType represents an error indicating that the column type for partitioning is not supported. +var ErrUnsupportedPartitionKeyType = errors.New("unsupported partition key column type") + +type ColumnType string + +type Column struct { + Schema string + Table string + Name string + DataType ColumnType +} + +func (c Column) String() string { + return strings.Join([]string{c.Schema, c.Table, c.Name}, ".") +} + +// Return the PostgreSQL data type of the specified column +func (p PostgreSQL) getColumnDataType(column Column) (ColumnType, error) { + var columnType string + + query := `SELECT + data_type as columnType + FROM information_schema.columns + WHERE + table_schema = $1 + AND table_name = $2 + AND column_name = $3` + + err := p.db.QueryRow(p.ctx, query, column.Schema, column.Table, column.Name).Scan(&columnType) + if err != nil { + return "", fmt.Errorf("failed to get %s column type: %w", column, err) + } + + switch columnType { + case "date": + return DateColumnType, nil + case "timestamp": + return DateTimeColumnType, nil + case "timestamp without time zone": + return DateTimeColumnType, nil + case "uuid": + return UUIDColumnType, nil + default: + return "", fmt.Errorf("%w: %s", ErrUnsupportedPartitionKeyType, columnType) + } +} diff --git a/internal/infra/postgresql/column_internal_test.go b/internal/infra/postgresql/column_internal_test.go new file mode 100644 index 0000000..73dc97a --- /dev/null +++ b/internal/infra/postgresql/column_internal_test.go @@ -0,0 +1,66 @@ +package postgresql + +import ( + "fmt" + "testing" + + "github.com/pashagolub/pgxmock/v3" + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/stretchr/testify/assert" +) + +func TestGetColumn(t *testing.T) { + column := Column{ + Schema: "public", + Table: "my_table", + Name: "my_column", + } + + const query = `SELECT data_type as columnType FROM information_schema.columns WHERE table_schema = \$1 AND table_name = \$2 AND column_name = \$3` + + mock, err := pgxmock.NewConn() + if err != nil { + fmt.Println("ERROR: Fail to initialize PostgreSQL mock: %w", err) + panic(err) + } + + logger, err := logger.New(false, "text") + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + panic(err) + } + + p := New(*logger, mock) + + testCases := []struct { + name string + postgreSQLcolumn string + dataType ColumnType + }{ + { + "Date", + "date", + DateColumnType, + }, + { + "Date time", + "timestamp without time zone", + DateTimeColumnType, + }, + { + "UUID", + "uuid", + UUIDColumnType, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mock.ExpectQuery(query).WithArgs(column.Schema, column.Table, column.Name).WillReturnRows(mock.NewRows([]string{"columnType"}).AddRow(tc.postgreSQLcolumn)) + dataType, err := p.getColumnDataType(column) + + assert.Nil(t, err, "getColumnDataType should succeed") + assert.Equal(t, dataType, tc.dataType, "Column type should match") + }) + } +} diff --git a/internal/infra/postgresql/column_test.go b/internal/infra/postgresql/column_test.go new file mode 100644 index 0000000..18b18f9 --- /dev/null +++ b/internal/infra/postgresql/column_test.go @@ -0,0 +1,32 @@ +package postgresql_test + +import ( + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "gotest.tools/assert" +) + +func TestColumnAttributes(t *testing.T) { + testCases := []struct { + name string + column postgresql.Column + expectedName string + }{ + { + name: "Public schema", + column: postgresql.Column{ + Schema: "public", + Table: "my_table", + Name: "my_column", + }, + expectedName: "public.my_table.my_column", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.column.String(), tc.expectedName, "Column name don't match") + }) + } +} diff --git a/internal/infra/postgresql/connection.go b/internal/infra/postgresql/connection.go new file mode 100644 index 0000000..90d9d1c --- /dev/null +++ b/internal/infra/postgresql/connection.go @@ -0,0 +1,33 @@ +// Package postgresql provides methods to interact with PostgreSQL internal resources (tables, columns, ...) +package postgresql + +import ( + "context" + "fmt" + "strconv" + + "github.com/jackc/pgx/v5" +) + +type ConnectionSettings struct { + URL string + StatementTimeout int + LockTimeout int +} + +func GetDatabaseConnection(c ConnectionSettings) (*pgx.Conn, error) { + config, err := pgx.ParseConfig(c.URL) + if err != nil { + return nil, fmt.Errorf("failed to initialize database connection: %w", err) + } + + config.RuntimeParams["statement_timeout"] = strconv.Itoa(c.StatementTimeout) + config.RuntimeParams["lock_timeout"] = strconv.Itoa(c.LockTimeout) + + conn, err := pgx.ConnectConfig(context.Background(), config) + if err != nil { + return nil, fmt.Errorf("failed to connect to the database: %w", err) + } + + return conn, nil +} diff --git a/internal/infra/postgresql/error.go b/internal/infra/postgresql/error.go new file mode 100644 index 0000000..ec74bdf --- /dev/null +++ b/internal/infra/postgresql/error.go @@ -0,0 +1,17 @@ +package postgresql + +import ( + "errors" + + "github.com/jackc/pgx/v5/pgconn" +) + +const ( + ObjectNotInPrerequisiteStatePostgreSQLErrorCode = "55000" +) + +func isPostgreSQLErrorCode(err error, errorCode string) bool { + var pgErr *pgconn.PgError + + return errors.As(err, &pgErr) && pgErr.Code == errorCode +} diff --git a/internal/infra/postgresql/error_internal_test.go b/internal/infra/postgresql/error_internal_test.go new file mode 100644 index 0000000..a55ef48 --- /dev/null +++ b/internal/infra/postgresql/error_internal_test.go @@ -0,0 +1,45 @@ +package postgresql + +import ( + "errors" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "gotest.tools/assert" +) + +var ErrGeneric = errors.New("a generic error") + +func TestPostgreSQLError(t *testing.T) { + testCases := []struct { + name string + error error + code string + expected bool + }{ + { + name: "ObjectNotInPrerequisiteState", + error: &pgconn.PgError{Code: "55000"}, + code: "55000", + expected: true, + }, + { + name: "Non match error code", + error: &pgconn.PgError{Code: "42"}, + code: "55000", + expected: false, + }, + { + name: "Generic error", + error: ErrGeneric, + code: "", + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, isPostgreSQLErrorCode(tc.error, tc.code), tc.expected, "Error code should match") + }) + } +} diff --git a/internal/infra/postgresql/partition.go b/internal/infra/postgresql/partition.go new file mode 100644 index 0000000..f681221 --- /dev/null +++ b/internal/infra/postgresql/partition.go @@ -0,0 +1,35 @@ +package postgresql + +import "fmt" + +type Partition struct { + ParentTable string + Schema string + Name string + LowerBound interface{} + UpperBound interface{} +} + +func (p Partition) String() string { + return p.QualifiedName() +} + +// QualifiedName returns the fully qualified name of the partition (format: .) +// This is recommended to avoid schema conflicts when querying PostgreSQL catalog tables +func (p Partition) QualifiedName() string { + return fmt.Sprintf("%s.%s", p.Schema, p.Name) +} + +func (p Partition) ToTable() Table { + return Table{ + Schema: p.Schema, + Name: p.Name, + } +} + +func (p Partition) GetParentTable() Table { + return Table{ + Schema: p.Schema, + Name: p.ParentTable, + } +} diff --git a/internal/infra/postgresql/partition_internal_test.go b/internal/infra/postgresql/partition_internal_test.go new file mode 100644 index 0000000..0649f18 --- /dev/null +++ b/internal/infra/postgresql/partition_internal_test.go @@ -0,0 +1,144 @@ +package postgresql + +import ( + "testing" + "time" + + "gotest.tools/assert" +) + +func TestParseBounds(t *testing.T) { + testCases := []struct { + name string + partition Partition + lowerbound time.Time + upperBound time.Time + }{ + { + "Date bounds", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "2025-03-02", + }, + time.Date(2024, 1, 1, 0, 0, 0, 0, time.Now().UTC().Location()), + time.Date(2025, 3, 2, 0, 0, 0, 0, time.Now().UTC().Location()), + }, + { + "Datetime bounds", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01 10:00:00", + UpperBound: "2025-02-03 12:53:00", + }, + time.Date(2024, 1, 1, 10, 0, 0, 0, time.Now().UTC().Location()), + time.Date(2025, 2, 3, 12, 53, 0, 0, time.Now().UTC().Location()), + }, + { + "UUIDv7 bounds", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + time.Date(2024, 1, 1, 0, 0, 0, 0, time.Now().UTC().Location()), + time.Date(2024, 1, 2, 0, 0, 0, 0, time.Now().UTC().Location()), + }, + { + "Native Time.time bounds", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: time.Date(2024, 1, 1, 10, 12, 5, 0, time.Now().UTC().Location()), + UpperBound: time.Date(2025, 2, 3, 12, 53, 35, 0, time.Now().UTC().Location()), + }, + time.Date(2024, 1, 1, 10, 12, 5, 0, time.Now().UTC().Location()), + time.Date(2025, 2, 3, 12, 53, 35, 0, time.Now().UTC().Location()), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + lowerBound, upperBound, err := parseBounds(tc.partition) + + assert.NilError(t, err, "Bounds parsing should succeed") + assert.Equal(t, lowerBound, tc.lowerbound, "LowerBound mismatch") + assert.Equal(t, upperBound, tc.upperBound, "UpperBound mismatch") + }) + } +} + +func TestParseInvalidBounds(t *testing.T) { + testCases := []struct { + name string + partition Partition + }{ + { + "UUID v1 upper bound", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 + UpperBound: "47568e76-fb49-11ee-b9c7-325096b39f47", // UUIDv1 + }, + }, + { + "UUID v1 lower bound", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "ad5dac7a-fb46-11ee-be67-325096b39f47", // UUIDv1 + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + }, + { + "Mix date format", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "2024-01-02 00:00:00", + }, + }, + { + "Mix date and UUIDv7", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + }, + { + "Mix date and UUIDv7", + Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := parseBounds(tc.partition) + + assert.ErrorContains(t, err, "partition bounds cannot be decoded") + }) + } +} + +func TestDebug(t *testing.T) { + partition := Partition{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01 10:00:00", + UpperBound: "2024-01-02 14:00:00", + } + _, _, err := parseBoundAsDateTime(partition) + assert.NilError(t, err) +} diff --git a/internal/infra/postgresql/partition_test.go b/internal/infra/postgresql/partition_test.go new file mode 100644 index 0000000..5f48ad6 --- /dev/null +++ b/internal/infra/postgresql/partition_test.go @@ -0,0 +1,483 @@ +package postgresql_test + +import ( + "testing" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "gotest.tools/assert" +) + +func TestPartitionAttributes(t *testing.T) { + testCases := []struct { + name string + partition postgresql.Partition + expectedName string + expectedTable postgresql.Table + expectedParentTable postgresql.Table + }{ + { + name: "Public schema", + partition: postgresql.Partition{ + ParentTable: "my_table", + Schema: "public", + Name: "my_table_2024_12_25", + }, + expectedName: "public.my_table_2024_12_25", + expectedTable: postgresql.Table{ + Schema: "public", + Name: "my_table_2024_12_25", + }, + expectedParentTable: postgresql.Table{ + Schema: "public", + Name: "my_table", + }, + }, + { + name: "Dashed table", + partition: postgresql.Partition{ + ParentTable: "my-table", + Schema: "api", + Name: "my-table_2024_w01", + }, + expectedName: "api.my-table_2024_w01", + expectedTable: postgresql.Table{ + Schema: "api", + Name: "my-table_2024_w01", + }, + expectedParentTable: postgresql.Table{ + Schema: "api", + Name: "my-table", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.partition.QualifiedName(), tc.expectedName, "Qualified name don't match") + assert.Equal(t, tc.partition.String(), tc.expectedName, "Partition name don't match") + assert.Equal(t, tc.partition.ToTable(), tc.expectedTable, "Table don't match") + assert.Equal(t, tc.partition.GetParentTable(), tc.expectedParentTable, "Parent table don't match") + }) + } +} + +func TestPartitionName(t *testing.T) { + testCases := []struct { + name string + partition postgresql.PartitionConfiguration + time time.Time + expected postgresql.Partition + }{ + { + name: "Daily partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.DailyInterval, + Retention: 7, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), + expected: postgresql.Partition{ + Schema: "public", + Name: "my_table_2024_01_30", + LowerBound: time.Date(2024, 0o1, 30, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 31, 0, 0, 0, 0, time.UTC), + }, + }, + { + name: "Monthly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.MonthlyInterval, + Retention: 7, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), + expected: postgresql.Partition{ + Schema: "public", + Name: "my_table_2024_01", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + { + name: "Weekly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.WeeklyInterval, + Retention: 7, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), + expected: postgresql.Partition{ + Schema: "public", + Name: "my_table_2024_w05", + LowerBound: time.Date(2024, 0o1, 29, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o5, 0, 0, 0, 0, time.UTC), + }, + }, + { + name: "Yearly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.YearlyInterval, + Retention: 7, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), + expected: postgresql.Partition{ + Schema: "public", + Name: "my_table_2024", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2025, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, _ := tc.partition.GeneratePartition(tc.time) + assert.Equal(t, tc.expected.Schema, result.Schema, "Schema don't match") + assert.Equal(t, tc.expected.Name, result.Name, "Table name don't match") + assert.Equal(t, tc.expected.LowerBound, result.LowerBound, "Lower bound don't match") + assert.Equal(t, tc.expected.UpperBound, result.UpperBound, "Upper bound don't match") + }) + } +} + +func TestRetentionTableNames(t *testing.T) { + testCases := []struct { + name string + partition postgresql.PartitionConfiguration + time time.Time + expected []postgresql.Partition + }{ + { + name: "Daily partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.DailyInterval, + Retention: 4, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 0o3, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2024_01_02", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o2, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o3, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_01_01", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o2, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2023_12_31", + ParentTable: "my_table", + LowerBound: time.Date(2023, 12, 31, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2023_12_30", + ParentTable: "my_table", + LowerBound: time.Date(2023, 12, 30, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2023, 12, 31, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "Monthly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.MonthlyInterval, + Retention: 3, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o2, 25, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2024_01", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2023_12", + ParentTable: "my_table", + LowerBound: time.Date(2023, 12, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2023_11", + ParentTable: "my_table", + LowerBound: time.Date(2023, 11, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2023, 12, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "Weekly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.WeeklyInterval, + Retention: 2, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 9, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2024_w01", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 8, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2023_w52", + ParentTable: "my_table", + LowerBound: time.Date(2023, 12, 25, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "Yearly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.YearlyInterval, + Retention: 2, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 9, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2023", + ParentTable: "my_table", + LowerBound: time.Date(2023, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2022", + ParentTable: "my_table", + LowerBound: time.Date(2022, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2023, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "No retention", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.WeeklyInterval, + Retention: 0, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 9, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tables, _ := tc.partition.GetRetentionPartitions(tc.time) + + assert.DeepEqual(t, tables, tc.expected) + }) + } +} + +func TestPreProvisionedTableNames(t *testing.T) { + testCases := []struct { + name string + partition postgresql.PartitionConfiguration + time time.Time + expected []postgresql.Partition + }{ + { + name: "Daily partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.DailyInterval, + Retention: 4, + PreProvisioned: 4, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2024, 0o1, 29, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2024_01_30", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 30, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 31, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_01_31", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 31, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_02_01", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o2, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_02_02", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o2, 0o2, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o3, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "Monthly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.MonthlyInterval, + Retention: 3, + PreProvisioned: 3, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2023, 11, 29, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2023_12", + ParentTable: "my_table", + LowerBound: time.Date(2023, 12, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_01", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_02", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o3, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "Weekly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.WeeklyInterval, + Retention: 2, + PreProvisioned: 2, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2023, 12, 20, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2023_w52", + ParentTable: "my_table", + LowerBound: time.Date(2023, 12, 25, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2024_w01", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2024, 0o1, 8, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "Yearly partition", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.YearlyInterval, + Retention: 2, + PreProvisioned: 2, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2023, 12, 20, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{ + { + Schema: "public", + Name: "my_table_2024", + ParentTable: "my_table", + LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2025, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, { + Schema: "public", + Name: "my_table_2025", + ParentTable: "my_table", + LowerBound: time.Date(2025, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + UpperBound: time.Date(2026, 0o1, 0o1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + { + name: "No PreProvisioned", + partition: postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.WeeklyInterval, + Retention: 2, + PreProvisioned: 0, + CleanupPolicy: postgresql.DropCleanupPolicy, + }, + time: time.Date(2023, 12, 20, 12, 53, 45, 100, time.UTC), + expected: []postgresql.Partition{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tables, _ := tc.partition.GetPreProvisionedPartitions(tc.time) + + assert.DeepEqual(t, tables, tc.expected) + }) + } +} diff --git a/internal/infra/postgresql/partitionconfiguration.go b/internal/infra/postgresql/partitionconfiguration.go new file mode 100644 index 0000000..ca4f474 --- /dev/null +++ b/internal/infra/postgresql/partitionconfiguration.go @@ -0,0 +1,150 @@ +package postgresql + +import ( + "errors" + "fmt" + "time" +) + +type ( + Interval string + CleanupPolicy string +) + +const ( + DailyInterval Interval = "daily" + WeeklyInterval Interval = "weekly" + MonthlyInterval Interval = "monthly" + YearlyInterval Interval = "yearly" + DropCleanupPolicy CleanupPolicy = "drop" + DetachCleanupPolicy CleanupPolicy = "detach" + daysInAweek int = 7 +) + +var ErrUnsupportedInterval = errors.New("unsupported partition interval") + +type PartitionConfiguration struct { + Schema string `mapstructure:"schema" validate:"required"` + Table string `mapstructure:"table" validate:"required"` + PartitionKey string `mapstructure:"partitionKey" validate:"required"` + Interval Interval `mapstructure:"interval" validate:"required,oneof=daily weekly monthly yearly"` + Retention int `mapstructure:"retention" validate:"required,gt=0"` + PreProvisioned int `mapstructure:"preProvisioned" validate:"required,gt=0"` + CleanupPolicy CleanupPolicy `mapstructure:"cleanupPolicy" validate:"required,oneof=drop detach"` +} + +func (p PartitionConfiguration) GeneratePartition(forDate time.Time) (Partition, error) { + var suffix string + + var lowerBound, upperBound any + + switch p.Interval { + case DailyInterval: + suffix = forDate.Format("2006_01_02") + lowerBound, upperBound = getDailyBounds(forDate) + case WeeklyInterval: + year, week := forDate.ISOWeek() + suffix = fmt.Sprintf("%d_w%02d", year, week) + lowerBound, upperBound = getWeeklyBounds(forDate) + case MonthlyInterval: + suffix = forDate.Format("2006_01") + lowerBound, upperBound = getMonthlyBounds(forDate) + case YearlyInterval: + suffix = forDate.Format("2006") + lowerBound, upperBound = getYearlyBounds(forDate) + default: + return Partition{}, ErrUnsupportedInterval + } + + partition := Partition{ + Schema: p.Schema, + ParentTable: p.Table, + Name: fmt.Sprintf("%s_%s", p.Table, suffix), + LowerBound: lowerBound, + UpperBound: upperBound, + } + + return partition, nil +} + +func (p PartitionConfiguration) GetRetentionPartitions(forDate time.Time) ([]Partition, error) { + partitions := make([]Partition, p.Retention) + + for i := 1; i <= p.Retention; i++ { + prevDate, err := p.getPrevDate(forDate, i) + if err != nil { + return nil, fmt.Errorf("could not compute previous date: %w", err) + } + + partition, err := p.GeneratePartition(prevDate) + if err != nil { + return nil, fmt.Errorf("could not generate partition: %w", err) + } + + partitions[i-1] = partition + } + + return partitions, nil +} + +func (p PartitionConfiguration) GetPreProvisionedPartitions(forDate time.Time) ([]Partition, error) { + partitions := make([]Partition, p.PreProvisioned) + + for i := 1; i <= p.PreProvisioned; i++ { + nextDate, err := p.getNextDate(forDate, i) + if err != nil { + return nil, fmt.Errorf("could not compute next date: %w", err) + } + + partition, err := p.GeneratePartition(nextDate) + if err != nil { + return nil, fmt.Errorf("could not generate partition: %w", err) + } + + partitions[i-1] = partition + } + + return partitions, nil +} + +func (p PartitionConfiguration) getPrevDate(forDate time.Time, i int) (t time.Time, err error) { + switch p.Interval { + case DailyInterval: + t = forDate.AddDate(0, 0, -i) + case WeeklyInterval: + t = forDate.AddDate(0, 0, -i*daysInAweek) + case MonthlyInterval: + year, month, _ := forDate.Date() + + t = time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, forDate.Location()) + case YearlyInterval: + year, _, _ := forDate.Date() + + t = time.Date(year-i, 1, 1, 0, 0, 0, 0, forDate.Location()) + default: + return time.Time{}, ErrUnsupportedInterval + } + + return t, nil +} + +func (p PartitionConfiguration) getNextDate(forDate time.Time, i int) (t time.Time, err error) { + switch p.Interval { + case DailyInterval: + t = forDate.AddDate(0, 0, i) + case WeeklyInterval: + t = forDate.AddDate(0, 0, i*daysInAweek) + case MonthlyInterval: + year, month, _ := forDate.Date() + + t = time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, forDate.Location()) + case YearlyInterval: + year, _, _ := forDate.Date() + + t = time.Date(year+i, 1, 1, 0, 0, 0, 0, forDate.Location()) + default: + return time.Time{}, ErrUnsupportedInterval + } + + return t, nil +} diff --git a/internal/infra/postgresql/partitionsettings.go b/internal/infra/postgresql/partitionsettings.go new file mode 100644 index 0000000..f7320b9 --- /dev/null +++ b/internal/infra/postgresql/partitionsettings.go @@ -0,0 +1,23 @@ +package postgresql + +type PartitionSettings struct { + Strategy PartitionStrategy + Key string + KeyType ColumnType +} + +func (p PartitionSettings) SupportedStrategy() bool { + return p.Strategy == RangePartitionStrategy +} + +func (p PartitionSettings) SupportedKeyDataType() bool { + switch p.KeyType { + case + DateColumnType, + DateTimeColumnType, + UUIDColumnType: + return true + } + + return false +} diff --git a/internal/infra/postgresql/partitionsettings_test.go b/internal/infra/postgresql/partitionsettings_test.go new file mode 100644 index 0000000..ce4df05 --- /dev/null +++ b/internal/infra/postgresql/partitionsettings_test.go @@ -0,0 +1,63 @@ +package postgresql_test + +import ( + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "gotest.tools/assert" +) + +func TestPartitionSettings(t *testing.T) { + var UnsupportedColumnType postgresql.ColumnType = "unsupported" + + testCases := []struct { + name string + partition postgresql.PartitionSettings + supportedKeyDataType bool + supportedStrategy bool + }{ + { + name: "Public", + partition: postgresql.PartitionSettings{ + Strategy: postgresql.RangePartitionStrategy, + KeyType: postgresql.DateColumnType, + }, + supportedStrategy: true, + supportedKeyDataType: true, + }, + { + name: "Unsupported list strategy", + partition: postgresql.PartitionSettings{ + Strategy: postgresql.ListPartitionStrategy, + KeyType: postgresql.DateColumnType, + }, + supportedKeyDataType: true, + supportedStrategy: false, + }, + { + name: "Unsupported hash strategy", + partition: postgresql.PartitionSettings{ + Strategy: postgresql.HashPartitionStrategy, + KeyType: postgresql.DateColumnType, + }, + supportedKeyDataType: true, + supportedStrategy: false, + }, + { + name: "Unsupported column type", + partition: postgresql.PartitionSettings{ + Strategy: postgresql.RangePartitionStrategy, + KeyType: UnsupportedColumnType, + }, + supportedKeyDataType: false, + supportedStrategy: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.partition.SupportedStrategy(), tc.supportedStrategy, "Supported strategy mismatch") + assert.Equal(t, tc.partition.SupportedKeyDataType(), tc.supportedKeyDataType, "Supported key data type mismatch") + }) + } +} diff --git a/internal/infra/postgresql/partitionstrategy.go b/internal/infra/postgresql/partitionstrategy.go new file mode 100644 index 0000000..44adc45 --- /dev/null +++ b/internal/infra/postgresql/partitionstrategy.go @@ -0,0 +1,9 @@ +package postgresql + +const ( + RangePartitionStrategy PartitionStrategy = "RANGE" + ListPartitionStrategy PartitionStrategy = "LIST" + HashPartitionStrategy PartitionStrategy = "HASH" +) + +type PartitionStrategy string diff --git a/internal/infra/postgresql/postgresql.go b/internal/infra/postgresql/postgresql.go new file mode 100644 index 0000000..058f6b0 --- /dev/null +++ b/internal/infra/postgresql/postgresql.go @@ -0,0 +1,366 @@ +package postgresql + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/qonto/postgresql-partition-manager/internal/infra/retry" + "github.com/qonto/postgresql-partition-manager/internal/infra/uuid7" +) + +// ErrUnsupportedPartitionStrategy represents an error indicating that the partitioning strategy on the table is not supported. +var ( + ErrUnsupportedPartitionStrategy = errors.New("unsupported partitioning strategy") + ErrPartitionNotFound = errors.New("partition not found") +) + +type PostgreSQL struct { + ctx context.Context + db PgxIface + logger slog.Logger +} + +type PgxIface interface { + Close(context.Context) error + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) + PgConn() *pgconn.PgConn +} + +func New(logger slog.Logger, db PgxIface) *PostgreSQL { + return &PostgreSQL{ + ctx: context.TODO(), + db: db, + logger: logger, + } +} + +func (p PostgreSQL) CreatePartition(partitionConfiguration PartitionConfiguration, partition Partition) error { + p.logger.Debug("Creating partition", "schema", partition.Schema, "table", partition.Name) + + tableExists, err := p.tableExists(partition.ToTable()) + if err != nil { + return fmt.Errorf("failed to check if table exists: %w", err) + } + + if !tableExists { + query := fmt.Sprintf("CREATE TABLE %s (LIKE %s)", partition.QualifiedName(), partition.ParentTable) + p.logger.Debug("Create table", "query", query) + + _, err := p.db.Exec(context.Background(), query) + if err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + p.logger.Info("Table created", "schema", partition.Schema, "table", partition.Name) + } else { + p.logger.Info("Table already exists, skip", "schema", partition.Schema, "table", partition.Name) + } + + lowerBoundTime, upperBoundTime, err := parseBounds(partition) + if err != nil { + return fmt.Errorf("failed to bounds: %w", err) + } + + partitionSettings, err := p.GetPartitionSettings(partition.GetParentTable()) + if err != nil { + return fmt.Errorf("failed to get partition settings: %w", err) + } + + switch partitionSettings.KeyType { + case DateColumnType: + partition.LowerBound = lowerBoundTime.Format("2006-01-02") + partition.UpperBound = upperBoundTime.Format("2006-01-02") + case DateTimeColumnType: + partition.LowerBound = lowerBoundTime.Format("2006-01-02 00:00:00") + partition.UpperBound = upperBoundTime.Format("2006-01-02 00:00:00") + case UUIDColumnType: + partition.LowerBound = uuid7.FromTime(lowerBoundTime) + partition.UpperBound = uuid7.FromTime(upperBoundTime) + default: + return ErrUnsupportedPartitionStrategy + } + + partitionAttached, err := p.isPartitionIsAttached(partition) + if err != nil { + return fmt.Errorf("failed to check partition attachment status: %w", err) + } + + if partitionAttached { + p.logger.Info("Table is already attached to the parent table, skip", "schema", partition.Schema, "table", partition.Name) + + return nil + } + + maxRetries := 3 + + err = retry.WithRetry(maxRetries, func(attempt int) error { + err := p.attachPartition(partition) + if err != nil { + p.logger.Warn("fail to attach partition", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) + } + + return err + }) + if err != nil { + return fmt.Errorf("failed to attach partition after retries: %w", err) + } + + p.logger.Info("Partition attached to parent table", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.GetParentTable().Name) + + return nil +} + +func (p PostgreSQL) tableExists(table Table) (bool, error) { + query := `SELECT EXISTS( + SELECT c.oid + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 + );` + + var exists bool + + err := p.db.QueryRow(context.Background(), query, table.Schema, table.Name).Scan(&exists) + if err != nil { + return false, fmt.Errorf("failed to get table: %w", err) + } + + return exists, nil +} + +func (p PostgreSQL) GetPartitionSettings(table Table) (PartitionSettings, error) { + var partkeydef []string + + maxRetries := 3 + + err := retry.WithRetry(maxRetries, func(attempt int) error { + // pg_get_partkeydef() is a system function returning the definition of a partitioning key + // It return a text string: () + // Example for RANGE (created_at) + query := fmt.Sprintf(` + SELECT regexp_match(partkeydef, '^(.*) \((.*)\)$') + FROM pg_catalog.pg_get_partkeydef('%s'::regclass) as partkeydef + `, table.QualifiedName()) + + err := p.db.QueryRow(p.ctx, query).Scan(&partkeydef) + if err != nil { + p.logger.Warn("failed to get partitioning key", "error", err, "schema", table.Schema, "table", table.Name, "attempt", attempt, "max_retries", maxRetries) + + return fmt.Errorf("failed to get partition key: %w", err) + } + + return nil + }) + if err != nil { + return PartitionSettings{}, fmt.Errorf("failed to get partitioning key after retries: %w", err) + } + + if len(partkeydef) == 0 { + return PartitionSettings{}, ErrPartitionNotFound + } + + settings := PartitionSettings{ + Key: partkeydef[1], + } + + rawPartitionStrategy := partkeydef[0] + switch rawPartitionStrategy { + case string(RangePartitionStrategy): + settings.Strategy = RangePartitionStrategy + case string(ListPartitionStrategy): + settings.Strategy = ListPartitionStrategy + case string(HashPartitionStrategy): + settings.Strategy = HashPartitionStrategy + default: + return settings, fmt.Errorf("%w: %s", ErrUnsupportedPartitionStrategy, rawPartitionStrategy) + } + + keyColumn := Column{ + Schema: table.Schema, + Table: table.Name, + Name: settings.Key, + } + + settings.KeyType, err = p.getColumnDataType(keyColumn) + if err != nil { + return settings, fmt.Errorf("failed to get partition key data type: %s: %w", rawPartitionStrategy, err) + } + + return settings, nil +} + +func (p PostgreSQL) isPartitionIsAttached(partition Partition) (bool, error) { + query := `SELECT EXISTS( + SELECT 1 FROM pg_inherits WHERE inhrelid = $1::regclass + )` + + var exists bool + + err := p.db.QueryRow(context.Background(), query, partition.QualifiedName()).Scan(&exists) + if err != nil { + return false, fmt.Errorf("failed to check partition attachment: %w", err) + } + + return exists, nil +} + +func (p PostgreSQL) attachPartition(partition Partition) error { + query := fmt.Sprintf("ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", partition.GetParentTable().QualifiedName(), partition.QualifiedName(), partition.LowerBound, partition.UpperBound) + p.logger.Debug("Attach partition", "query", query, "partition", partition.Name, "table", partition.GetParentTable().Name) + + _, err := p.db.Exec(context.Background(), query) + if err != nil { + return fmt.Errorf("failed to attach partition: %w", err) + } + + return nil +} + +// Function detachPartitionConcurrently detaches specified partition from the parent table. +// The partition exists as standalone table after detaching. +// More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION +func (p PostgreSQL) detachPartitionConcurrently(partition Partition) error { + query := fmt.Sprintf(`ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY;`, partition.GetParentTable().Name, partition.QualifiedName()) + p.logger.Debug("Detach partition", "schema", partition.Schema, "table", partition.Name, "query", query, "parent_table", partition.GetParentTable().Name) + + _, err := p.db.Exec(context.Background(), query) + if err != nil { + return fmt.Errorf("failed to detach partition from the parent table: %w", err) + } + + return nil +} + +// Function finalizePartitionDetach finalize a partition detach operation. +// It's required when a partition is in "detach pending" status. +// More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION +func (p PostgreSQL) finalizePartitionDetach(partition Partition) error { + query := fmt.Sprintf(`ALTER TABLE %s DETACH PARTITION %s FINALIZE;`, partition.GetParentTable().Name, partition.QualifiedName()) + p.logger.Debug("finialize detach partition", "schema", partition.Schema, "table", partition.Name, "query", query, "parent_table", partition.GetParentTable().Name) + + _, err := p.db.Exec(context.Background(), query) + if err != nil { + return fmt.Errorf("failed to finalize partition detach: %w", err) + } + + return nil +} + +func (p PostgreSQL) dropTable(table Table) error { + query := fmt.Sprintf("DROP TABLE %s", table.QualifiedName()) + p.logger.Debug("Drop table", "schema", table.Schema, "table", table.Name, "query", query) + + _, err := p.db.Exec(context.Background(), query) + if err != nil { + return fmt.Errorf("failed to drop table: %w", err) + } + + return nil +} + +func (p PostgreSQL) ListPartitions(table Table) (partitions []Partition, err error) { + query := fmt.Sprintf(` + WITH parts as ( + SELECT + relnamespace::regnamespace as schema, + c.oid::pg_catalog.regclass AS part_name, + regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FOR VALUES FROM \(''(.*)''\) TO \(''(.*)''\)') AS bounds + FROM + pg_catalog.pg_class c JOIN pg_catalog.pg_inherits i ON (c.oid = i.inhrelid) + WHERE i.inhparent = '%s'::regclass + AND c.relkind='r' + ) + SELECT + schema, + part_name as name, + '%s' as parentTable, + bounds[1]::text AS lower_bound, + bounds[2]::text AS upper_bound + FROM parts + ORDER BY part_name;`, table.QualifiedName(), table.Name) + + rows, err := p.db.Query(context.Background(), query) + if err != nil { + return nil, fmt.Errorf("failed to list partitions: %w", err) + } + + rawPartitions, err := pgx.CollectRows(rows, pgx.RowToStructByName[Partition]) + if err != nil { + return nil, fmt.Errorf("failed to cast list: %w", err) + } + + for _, partition := range rawPartitions { + lowerBound, upperBound, err := parseBounds(partition) + if err != nil { + return nil, fmt.Errorf("failed to parse bounds: %w", err) + } + + partition.LowerBound = lowerBound + partition.UpperBound = upperBound + + partitions = append(partitions, partition) + } + + return partitions, nil +} + +func (p PostgreSQL) DetachPartition(partition Partition) error { + p.logger.Debug("Detach partition", "schema", partition.Schema, "table", partition.Name) + + maxRetries := 3 + + err := retry.WithRetry(maxRetries, func(attempt int) error { + err := p.detachPartitionConcurrently(partition) + if err != nil { + // detachPartitionConcurrently() could fail if the specified partition is in pending detach status + // It could occurred when a previous detach partition concurrently operation was canceled or interrupted + // It prevent any other detach operations on the table + // More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION + // To unblock the situation, we try to finalize the detach operation on Object Not In Prerequisite State error + if isPostgreSQLErrorCode(err, ObjectNotInPrerequisiteStatePostgreSQLErrorCode) { + p.logger.Warn("Table is already pending detach in partitioned, retry with finalize", "error", err, "schema", partition.Schema, "table", partition.Name) + + finalizeErr := p.finalizePartitionDetach(partition) + if finalizeErr == nil { + err = nil // Returns a success since the partition detach operation has been completed + } + } else { + p.logger.Warn("Fail to detach partition", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) + } + } + + return err + }) + if err != nil { + return fmt.Errorf("failed to detach partition after retries: %w", err) + } + + return nil +} + +func (p PostgreSQL) DeletePartition(partition Partition) error { + p.logger.Debug("Deleting partition", "schema", partition.Schema, "table", partition.Name) + + maxRetries := 3 + + err := retry.WithRetry(maxRetries, func(attempt int) error { + err := p.dropTable(partition.ToTable()) + if err != nil { + p.logger.Warn("Fail to drop table", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) + } + + return err + }) + if err != nil { + return fmt.Errorf("failed to drop table after retries: %w", err) + } + + return nil +} diff --git a/internal/infra/postgresql/postgresql_internal_test.go b/internal/infra/postgresql/postgresql_internal_test.go new file mode 100644 index 0000000..8f9d929 --- /dev/null +++ b/internal/infra/postgresql/postgresql_internal_test.go @@ -0,0 +1,272 @@ +package postgresql + +import ( + "fmt" + "testing" + + "github.com/pashagolub/pgxmock/v3" + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/stretchr/testify/assert" +) + +func getMock(t *testing.T) (pgxmock.PgxConnIface, *PostgreSQL) { + t.Helper() + + mock, err := pgxmock.NewConn() + if err != nil { + fmt.Println("ERROR: Fail to initialize PostgreSQL mock: %w", err) + panic(err) + } + + logger, err := logger.New(false, "text") + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + panic(err) + } + + p := New(*logger, mock) + + return mock, p +} + +func TestTableExists(t *testing.T) { + mock, p := getMock(t) + + existingTable := Table{Schema: "public", Name: "my_table"} + existingTables := []Table{existingTable} + + query := `SELECT EXISTS\( SELECT c.oid FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = \$1 AND c.relname = \$2 \)` + for _, table := range existingTables { + mock.ExpectQuery(query).WithArgs(table.Schema, table.Name).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(true)) + } + + tableWithSameNameInDifferentSchema := Table{Schema: "another_schema", Name: "my_table"} + tableWithSameSchemaButDifferentName := Table{Schema: "public", Name: "another_table"} + missingTables := []Table{tableWithSameNameInDifferentSchema, tableWithSameSchemaButDifferentName} + + for _, table := range missingTables { + mock.ExpectQuery(query).WithArgs(table.Schema, table.Name).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(false)) + } + + testCases := []struct { + name string + table Table + exists bool + }{ + { + "Existing table", + existingTable, + true, + }, + { + "Table with same name in different schema", + tableWithSameNameInDifferentSchema, + false, + }, + { + "Table with same schema, but different name", + tableWithSameSchemaButDifferentName, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + exists, err := p.tableExists(tc.table) + + assert.Nil(t, err, "tableExists should succeed") + assert.Equal(t, exists, tc.exists, "Exists should match") + }) + } +} + +func TestIsPartitionIsAttached(t *testing.T) { + mock, p := getMock(t) + + attachedPartition := Partition{ParentTable: "partioned_table", Schema: "public", Name: "partioned_table_2024"} + unattachedPartition := Partition{ParentTable: "partioned_table", Schema: "public", Name: "partioned_table_1999"} + + query := `SELECT EXISTS\( SELECT 1 FROM pg_inherits WHERE inhrelid = \$1::regclass \)` + mock.ExpectQuery(query).WithArgs(attachedPartition.QualifiedName()).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(true)) + mock.ExpectQuery(query).WithArgs(unattachedPartition.QualifiedName()).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(false)) + + testCases := []struct { + name string + partition Partition + exists bool + }{ + { + "Attached partition", + attachedPartition, + true, + }, + { + "Unattached partition", + unattachedPartition, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + exists, err := p.isPartitionIsAttached(tc.partition) + + assert.Nil(t, err, "isPartitionIsAttached should succeed") + assert.Equal(t, exists, tc.exists, "Exists should match") + }) + } +} + +func TestDropTable(t *testing.T) { + mock, p := getMock(t) + + table := Table{Schema: "public", Name: "my_table"} + + query := fmt.Sprintf("DROP TABLE %s", table.QualifiedName()) + mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("DROP", 1)) + + err := p.dropTable(table) + assert.Nil(t, err, "dropTable should succeed") +} + +func TestGetPartitionSettings(t *testing.T) { + mock, p := getMock(t) + + queryPartitionSettings := `SELECT regexp_match\(partkeydef` // Partial query + queryColumn := `SELECT data_type as columnType FROM information_schema.columns WHERE table_schema = \$1 AND table_name = \$2 AND column_name = \$3` + + testCases := []struct { + name string + column Column + expectedSettings PartitionSettings + }{ + { + "Date partition", + Column{ + Schema: "public", + Table: "my_table", + Name: "created_at", + DataType: DateColumnType, + }, + PartitionSettings{ + Strategy: RangePartitionStrategy, + Key: "created_at", + KeyType: DateColumnType, + }, + }, + { + "Datetime partition", + Column{ + Schema: "public", + Table: "my_table", + Name: "hour", + DataType: DateTimeColumnType, + }, + PartitionSettings{ + Strategy: RangePartitionStrategy, + Key: "hour", + KeyType: DateTimeColumnType, + }, + }, + { + "UUID partition", + Column{ + Schema: "public", + Table: "my_table", + Name: "id", + DataType: UUIDColumnType, + }, + PartitionSettings{ + Strategy: RangePartitionStrategy, + Key: "id", + KeyType: UUIDColumnType, + }, + }, + { + "List partition", + Column{ + Schema: "public", + Table: "my_table", + Name: "created_at", + DataType: UUIDColumnType, + }, + PartitionSettings{ + Strategy: ListPartitionStrategy, + Key: "created_at", + KeyType: UUIDColumnType, + }, + }, + { + "Hash partition", + Column{ + Schema: "public", + Table: "my_table", + Name: "created_at", + DataType: UUIDColumnType, + }, + PartitionSettings{ + Strategy: HashPartitionStrategy, + Key: "created_at", + KeyType: UUIDColumnType, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"partkeydef"}).AddRow([]string{string(tc.expectedSettings.Strategy), tc.column.Name})) + mock.ExpectQuery(queryColumn).WithArgs(tc.column.Schema, tc.column.Table, tc.column.Name).WillReturnRows(mock.NewRows([]string{"columnType"}).AddRow(string(tc.column.DataType))) + + table := Table{Schema: tc.column.Schema, Name: tc.column.Table} + settings, err := p.GetPartitionSettings(table) + + assert.Nil(t, err, "GetPartitionSettings should succeed") + assert.Equal(t, settings, tc.expectedSettings, "partition settings should succeed") + }) + } +} + +func TestGetPartitionSettingsErrors(t *testing.T) { + mock, p := getMock(t) + + queryPartitionSettings := `SELECT regexp_match\(partkeydef` // Partial query + queryColumn := `SELECT data_type as columnType FROM information_schema.columns WHERE table_schema = \$1 AND table_name = \$2 AND column_name = \$3` + + testCases := []struct { + name string + column Column + expectedSettings PartitionSettings + }{ + { + "Missing partition", + Column{ + Schema: "public", + Table: "my_table", + Name: "created_at", + DataType: DateColumnType, + }, + PartitionSettings{ + Strategy: RangePartitionStrategy, + Key: "created_at", + KeyType: DateColumnType, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + maxRetries := 3 + for attempt := 1; attempt <= maxRetries; attempt++ { + mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"x"})) + mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"x"})) + mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"x"})) + } + mock.ExpectQuery(queryColumn).WithArgs(tc.column.Schema, tc.column.Table, tc.column.Name).WillReturnRows(mock.NewRows([]string{"x"})) + + table := Table{Schema: tc.column.Schema, Name: tc.column.Table} + _, err := p.GetPartitionSettings(table) + + assert.Error(t, err, ErrPartitionNotFound) + }) + } +} diff --git a/internal/infra/postgresql/server.go b/internal/infra/postgresql/server.go new file mode 100644 index 0000000..60f3411 --- /dev/null +++ b/internal/infra/postgresql/server.go @@ -0,0 +1,39 @@ +package postgresql + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "time" +) + +var ErrUnkownServerVersion = errors.New("could not find server version") + +func (p *PostgreSQL) GetVersion() (int64, error) { + serverVersionStr := p.db.PgConn().ParameterStatus("server_version") + serverVersionStr = regexp.MustCompile(`^[0-9]+`).FindString(serverVersionStr) + + if serverVersionStr == "" { + return 0, ErrUnkownServerVersion + } + + serverVersion, err := strconv.ParseInt(serverVersionStr, 10, 64) + if err != nil { + return 0, ErrUnkownServerVersion + } + + return serverVersion, nil +} + +func (p *PostgreSQL) GetServerTime() (time.Time, error) { + var serverTime time.Time + + err := p.db.QueryRow(context.Background(), "SELECT NOW() AT TIME ZONE 'UTC' as serverTime").Scan(&serverTime) + if err != nil { + return time.Time{}, fmt.Errorf("could not get server time: %w", err) + } + + return serverTime, nil +} diff --git a/internal/infra/postgresql/server_test.go b/internal/infra/postgresql/server_test.go new file mode 100644 index 0000000..7f4a0f1 --- /dev/null +++ b/internal/infra/postgresql/server_test.go @@ -0,0 +1,36 @@ +package postgresql_test + +import ( + "fmt" + "testing" + "time" + + "github.com/pashagolub/pgxmock/v3" + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/stretchr/testify/assert" +) + +func TestGetServerTime(t *testing.T) { + mock, err := pgxmock.NewConn() + if err != nil { + fmt.Println("ERROR: Fail to initialize PostgreSQL mock: %w", err) + panic(err) + } + + currrentTime := time.Now() + query := `SELECT NOW\(\) AT TIME ZONE \'UTC\' as serverTime` + mock.ExpectQuery(query).WillReturnRows(mock.NewRows([]string{"serverTime"}).AddRow(currrentTime)) + + logger, err := logger.New(false, "text") + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + panic(err) + } + + p := postgresql.New(*logger, mock) + serverTime, err := p.GetServerTime() + + assert.Nil(t, err, "GetServerTime should succeed") + assert.Equal(t, serverTime, currrentTime, "Time should match") +} diff --git a/internal/infra/postgresql/table.go b/internal/infra/postgresql/table.go new file mode 100644 index 0000000..40c4699 --- /dev/null +++ b/internal/infra/postgresql/table.go @@ -0,0 +1,18 @@ +package postgresql + +import "fmt" + +type Table struct { + Schema string + Name string +} + +func (t Table) String() string { + return t.QualifiedName() +} + +// QualifiedName returns the fully qualified name of the table (format: .
) +// This is recommended to avoid schema conflicts when querying PostgreSQL catalog tables +func (t Table) QualifiedName() string { + return fmt.Sprintf("%s.%s", t.Schema, t.Name) +} diff --git a/internal/infra/postgresql/table_test.go b/internal/infra/postgresql/table_test.go new file mode 100644 index 0000000..08ee7e6 --- /dev/null +++ b/internal/infra/postgresql/table_test.go @@ -0,0 +1,32 @@ +package postgresql_test + +import ( + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/stretchr/testify/assert" +) + +func TestTableAttributes(t *testing.T) { + testCases := []struct { + name string + table postgresql.Table + expectedName string + }{ + { + name: "Public schema", + table: postgresql.Table{ + Schema: "public", + Name: "my_table", + }, + expectedName: "public.my_table", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.table.QualifiedName(), tc.expectedName, "Table name name don't match") + assert.Equal(t, tc.table.String(), tc.expectedName, "Table name name don't match") + }) + } +} From d08c93fdf840726aa3a77b3a117198f69cadde7a Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:18:05 +0200 Subject: [PATCH 10/27] chore(go): Add PPM package --- pkg/ppm/check.go | 173 ++++++++++++++++++++++++++ pkg/ppm/check_test.go | 198 ++++++++++++++++++++++++++++++ pkg/ppm/cleanup.go | 66 ++++++++++ pkg/ppm/cleanup_test.go | 152 +++++++++++++++++++++++ pkg/ppm/mocks/PostgreSQLClient.go | 170 +++++++++++++++++++++++++ pkg/ppm/ppm.go | 60 +++++++++ pkg/ppm/provisioning.go | 53 ++++++++ pkg/ppm/provisioning_test.go | 121 ++++++++++++++++++ pkg/ppm/server.go | 80 ++++++++++++ pkg/ppm/server_test.go | 66 ++++++++++ 10 files changed, 1139 insertions(+) create mode 100644 pkg/ppm/check.go create mode 100644 pkg/ppm/check_test.go create mode 100644 pkg/ppm/cleanup.go create mode 100644 pkg/ppm/cleanup_test.go create mode 100644 pkg/ppm/mocks/PostgreSQLClient.go create mode 100644 pkg/ppm/ppm.go create mode 100644 pkg/ppm/provisioning.go create mode 100644 pkg/ppm/provisioning_test.go create mode 100644 pkg/ppm/server.go create mode 100644 pkg/ppm/server_test.go diff --git a/pkg/ppm/check.go b/pkg/ppm/check.go new file mode 100644 index 0000000..731361f --- /dev/null +++ b/pkg/ppm/check.go @@ -0,0 +1,173 @@ +package ppm + +import ( + "errors" + "fmt" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +var ( + ErrUnsupportedKeyDataType = errors.New("unsupported partitioning column type on the table") + ErrUnsupportedPartitionStrategy = errors.New("unsupported partitioning strategy on the table") + ErrPartitionKeyMismatch = errors.New("mismatch of partition keys between parameters and table") + ErrUnexpectedOrMissingPartitions = errors.New("unexpected or missing partitions") + ErrInvalidPartitionConfiguration = errors.New("at least one partition contains an invalid configuration") +) + +func (p *PPM) CheckPartitions() error { + partitionContainsAnError := false + + for name, config := range p.partitions { + p.logger.Info("Checking partition", "partition", name) + + err := p.checkPartition(config) + if err != nil { + partitionContainsAnError = true + + p.logger.Error(err.Error(), "schema", config.Schema, "table", config.Table) + } + } + + if partitionContainsAnError { + return ErrInvalidPartitionConfiguration + } + + return nil +} + +func (p *PPM) checkPartition(config postgresql.PartitionConfiguration) error { + p.logger.Debug("Checking partition", "schema", config.Schema, "table", config.Table) + + err := p.checkPartitionKey(config) + if err != nil { + return fmt.Errorf("failed to check partition key: %w", err) + } + + err = p.checkPartitionsConfiguration(config) + if err != nil { + return fmt.Errorf("failed to check partitions configuration: %w", err) + } + + p.logger.Debug("Partitions match the configuration", "schema", config.Schema, "table", config.Table) + + return nil +} + +func (p *PPM) checkPartitionKey(config postgresql.PartitionConfiguration) error { + partition := postgresql.Partition{ + Schema: config.Schema, + ParentTable: config.Table, + Name: config.Table, + } + + partitionSettings, err := p.db.GetPartitionSettings(partition.GetParentTable()) + if err != nil { + return fmt.Errorf("failed to get partition settings: %w", err) + } + + p.logger.Debug("Partition configuration found", "schema", partition.Schema, "table", partition.Name, "partition_key", partitionSettings.Key, "partition_key_type", partitionSettings.KeyType, "partition_strategy", partitionSettings.Strategy) + + if partitionSettings.Key != config.PartitionKey { + p.logger.Warn("Partition key mismatch", "expected", config.PartitionKey, "current", partitionSettings.Key) + + return ErrPartitionKeyMismatch + } + + if !partitionSettings.SupportedStrategy() { + return ErrUnsupportedPartitionStrategy + } + + if !partitionSettings.SupportedKeyDataType() { + return ErrUnsupportedKeyDataType + } + + return nil +} + +func (p *PPM) comparePartitions(existingTables, expectedTables []postgresql.Partition) (unexpectedTables, missingTables, incorrectBounds []postgresql.Partition) { + // Maps for tracking presence + existing := make(map[string]postgresql.Partition) + expectedAndExists := make(map[string]bool) + + for _, t := range existingTables { + existing[t.Name] = t + } + + for _, t := range expectedTables { + if _, found := existing[t.Name]; found { + expectedAndExists[t.Name] = true + incorrectBound := false + + if existing[t.Name].UpperBound != t.UpperBound { + incorrectBound = true + + p.logger.Warn("Incorrect upper partition bound", "schema", t.Schema, "table", t.Name, "current_bound", existing[t.Name].UpperBound, "expected_bound", t.UpperBound) + } + + if existing[t.Name].LowerBound != t.LowerBound { + incorrectBound = true + + p.logger.Warn("Incorrect lower partition bound", "schema", t.Schema, "table", t.Name, "current_bound", existing[t.Name].LowerBound, "expected_bound", t.LowerBound) + } + + if incorrectBound { + incorrectBounds = append(incorrectBounds, t) + } + } else { + missingTables = append(missingTables, t) + } + } + + for _, t := range existingTables { + if _, found := expectedAndExists[t.Name]; !found { + // Only in existingTables and not in both + unexpectedTables = append(unexpectedTables, t) + } + } + + return unexpectedTables, missingTables, incorrectBounds +} + +func (p *PPM) checkPartitionsConfiguration(partition postgresql.PartitionConfiguration) error { + partitionContainAnError := false + + currentTime := time.Now() + + expectedPartitions, err := getExpectedPartitions(partition, currentTime) + if err != nil { + return fmt.Errorf("could not generate expected partitions: %w", err) + } + + foundPartitions, err := p.db.ListPartitions(postgresql.Table{Schema: partition.Schema, Name: partition.Table}) + if err != nil { + return fmt.Errorf("could not list partitions: %w", err) + } + + unexpected, missing, incorrectBound := p.comparePartitions(foundPartitions, expectedPartitions) + + if len(unexpected) > 0 { + partitionContainAnError = true + + p.logger.Warn("Found unexpected tables", "tables", unexpected) + } + + if len(missing) > 0 { + partitionContainAnError = true + + p.logger.Warn("Found missing tables", "tables", missing) + } + + if len(incorrectBound) > 0 { + partitionContainAnError = true + + p.logger.Warn("Found partitions with incorrect bound", "tables", incorrectBound) + } + + if partitionContainAnError { + return ErrUnexpectedOrMissingPartitions + } + + return nil +} diff --git a/pkg/ppm/check_test.go b/pkg/ppm/check_test.go new file mode 100644 index 0000000..5f4f8e0 --- /dev/null +++ b/pkg/ppm/check_test.go @@ -0,0 +1,198 @@ +package ppm_test + +import ( + "context" + "fmt" + "log/slog" + "testing" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/pkg/ppm" + "github.com/qonto/postgresql-partition-manager/pkg/ppm/mocks" + "gotest.tools/assert" +) + +var ( + dayBeforeYesterday = yesterday.AddDate(0, 0, -1) + yesterday = time.Now().AddDate(0, 0, -1) + today = time.Now() + tomorrow = time.Now().AddDate(0, 0, +1) + dayAfterTomorrow = tomorrow.AddDate(0, 0, +1) +) + +func getTestMocks(t *testing.T) (*slog.Logger, *mocks.PostgreSQLClient) { + t.Helper() + + logger, err := logger.New(false, "text") + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + panic(err) + } + + postgreSQLMock := mocks.PostgreSQLClient{} + + return logger, &postgreSQLMock +} + +func TestCheckPartitions(t *testing.T) { + logger, postgreSQLMock := getTestMocks(t) + + partitions := map[string]postgresql.PartitionConfiguration{} + partitions["daily partition"] = postgresql.PartitionConfiguration{Schema: "app", Table: "daily_table1", PartitionKey: "column", Interval: postgresql.DailyInterval, Retention: 2, PreProvisioned: 2} + partitions["daily partition without retention"] = postgresql.PartitionConfiguration{Schema: "public", Table: "daily_table2", PartitionKey: "created_at", Interval: postgresql.DailyInterval, Retention: 0, PreProvisioned: 1} + partitions["daily partition without preprovisioned"] = postgresql.PartitionConfiguration{Schema: "public", Table: "daily_table3", PartitionKey: "column", Interval: postgresql.DailyInterval, Retention: 4, PreProvisioned: 0} + partitions["weekly partition"] = postgresql.PartitionConfiguration{Schema: "public", Table: "weekly_table", PartitionKey: "weekly", Interval: postgresql.WeeklyInterval, Retention: 2, PreProvisioned: 2} + partitions["monthly partition"] = postgresql.PartitionConfiguration{Schema: "public", Table: "monthly_table", PartitionKey: "month", Interval: postgresql.MonthlyInterval, Retention: 2, PreProvisioned: 2} + partitions["yearly partition"] = postgresql.PartitionConfiguration{Schema: "public", Table: "yearly_table", PartitionKey: "year", Interval: postgresql.YearlyInterval, Retention: 4, PreProvisioned: 4} + + // Build mock for each partitions + for _, p := range partitions { + settings := postgresql.PartitionSettings{ + KeyType: postgresql.DateColumnType, + Strategy: postgresql.RangePartitionStrategy, + Key: p.PartitionKey, + } + postgreSQLMock.On("GetPartitionSettings", postgresql.Table{Schema: p.Schema, Name: p.Table}).Return(settings, nil).Once() + + var tables []postgresql.Partition + + var partition postgresql.Partition + + for i := 0; i <= p.Retention; i++ { + switch p.Interval { + case postgresql.DailyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i)) + case postgresql.WeeklyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i*7)) + case postgresql.MonthlyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(0, -i, 0)) + case postgresql.YearlyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(-i, 0, 0)) + default: + t.Errorf("unuspported partition interval in retention table mock") + } + + tables = append(tables, partition) + } + + for i := 0; i <= p.PreProvisioned; i++ { + switch p.Interval { + case postgresql.DailyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i)) + case postgresql.WeeklyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i*7)) + case postgresql.MonthlyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(0, i, 0)) + case postgresql.YearlyInterval: + partition, _ = p.GeneratePartition(time.Now().AddDate(i, 0, 0)) + default: + t.Errorf("unuspported partition interval in preprovisonned table mock") + } + + tables = append(tables, partition) + } + postgreSQLMock.On("ListPartitions", postgresql.Table{Schema: p.Schema, Name: p.Table}).Return(tables, nil).Once() + } + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, partitions) + assert.NilError(t, checker.CheckPartitions(), "Partitions should succeed") +} + +func TestCheckMissingPartitions(t *testing.T) { + logger, postgreSQLMock := getTestMocks(t) + + partition := postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.DailyInterval, + Retention: 2, + PreProvisioned: 2, + } + + todayPartition, _ := partition.GeneratePartition(today) + yesterdayPartition, _ := partition.GeneratePartition(yesterday) + tomorrowPartition, _ := partition.GeneratePartition(tomorrow) + + testCases := []struct { + name string + tables []postgresql.Partition + }{ + { + "Missing Yesterday retention partition", + []postgresql.Partition{ + todayPartition, + yesterdayPartition, + }, + }, + { + "Missing Tomorrow partition", + []postgresql.Partition{ + todayPartition, + tomorrowPartition, + }, + }, + { + "Missing Today partition", + []postgresql.Partition{ + yesterdayPartition, + tomorrowPartition, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + settings := postgresql.PartitionSettings{ + KeyType: postgresql.DateColumnType, + Strategy: postgresql.RangePartitionStrategy, + Key: partition.PartitionKey, + } + postgreSQLMock.On("GetPartitionSettings", postgresql.Table{Schema: partition.Schema, Name: partition.Table}).Return(settings, nil).Once() + + fmt.Println("tc.tables", tc.tables) + postgreSQLMock.On("ListPartitions", postgresql.Table{Schema: partition.Schema, Name: partition.Table}).Return(tc.tables, nil).Once() + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, map[string]postgresql.PartitionConfiguration{"test": partition}) + assert.Error(t, checker.CheckPartitions(), "at least one partition contains an invalid configuration") + }) + } +} + +func TestUnsupportedPartitionsStrategy(t *testing.T) { + logger, postgreSQLMock := getTestMocks(t) + + partition := postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: postgresql.DailyInterval, + Retention: 2, + PreProvisioned: 2, + } + + testCases := []struct { + name string + settings postgresql.PartitionSettings + }{ + { + "Unsupported list partition strategy", + postgresql.PartitionSettings{Strategy: postgresql.ListPartitionStrategy, Key: "created_at", KeyType: postgresql.DateColumnType}, + }, + { + "Unsupported hash partition strategy", + postgresql.PartitionSettings{Strategy: postgresql.HashPartitionStrategy, Key: "created_at", KeyType: postgresql.DateColumnType}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + postgreSQLMock.On("GetPartitionSettings", postgresql.Table{Schema: partition.Schema, Name: partition.Table}).Return(tc.settings, nil).Once() + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, map[string]postgresql.PartitionConfiguration{"test": partition}) + assert.Error(t, checker.CheckPartitions(), "at least one partition contains an invalid configuration") + }) + } +} diff --git a/pkg/ppm/cleanup.go b/pkg/ppm/cleanup.go new file mode 100644 index 0000000..557eff1 --- /dev/null +++ b/pkg/ppm/cleanup.go @@ -0,0 +1,66 @@ +package ppm + +import ( + "errors" + "fmt" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +var ErrPartitionCleanupFailed = errors.New("at least one partition contains could not be cleaned") + +func (p PPM) CleanupPartitions() error { + currentTime := time.Now() + partitionContainAnError := false + + for name, config := range p.partitions { + p.logger.Info("Cleaning partition", "partition", name) + + expectedPartitions, err := getExpectedPartitions(config, currentTime) + if err != nil { + return fmt.Errorf("could not generate expected partitions: %w", err) + } + + parentTable := postgresql.Table{Schema: config.Schema, Name: config.Table} + + foundPartitions, err := p.db.ListPartitions(parentTable) + if err != nil { + return fmt.Errorf("could not list partitions: %w", err) + } + + unexpected, _, _ := p.comparePartitions(foundPartitions, expectedPartitions) + + for _, partition := range unexpected { + err := p.db.DetachPartition(partition) + if err != nil { + partitionContainAnError = true + + p.logger.Error("Failed to detach partition", "schema", partition.Schema, "table", partition.Name, "error", err) + + continue + } + + p.logger.Info("Partition detached", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.GetParentTable().Name) + + if config.CleanupPolicy == postgresql.DropCleanupPolicy { + err := p.db.DeletePartition(partition) + if err != nil { + partitionContainAnError = true + + p.logger.Error("Failed to delete partition", "schema", partition.Schema, "table", partition.Name, "error", err) + + continue + } + + p.logger.Info("Partition deleted", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.GetParentTable().Name) + } + } + } + + if partitionContainAnError { + return ErrPartitionCleanupFailed + } + + return nil +} diff --git a/pkg/ppm/cleanup_test.go b/pkg/ppm/cleanup_test.go new file mode 100644 index 0000000..87dfef5 --- /dev/null +++ b/pkg/ppm/cleanup_test.go @@ -0,0 +1,152 @@ +package ppm_test + +import ( + "context" + "errors" + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/pkg/ppm" + "github.com/stretchr/testify/assert" +) + +var OneDayPartitionConfiguration = postgresql.PartitionConfiguration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: "daily", + Retention: 1, + PreProvisioned: 1, + CleanupPolicy: "drop", +} + +var ErrFake = errors.New("fake error") + +func TestCleanupPartitions(t *testing.T) { + yearBeforePartition, _ := OneDayPartitionConfiguration.GeneratePartition(dayBeforeYesterday.AddDate(-1, 0, 0)) + dayBeforeYesterdayPartition, _ := OneDayPartitionConfiguration.GeneratePartition(dayBeforeYesterday) + yesterdayPartition, _ := OneDayPartitionConfiguration.GeneratePartition(yesterday) + currentPartition, _ := OneDayPartitionConfiguration.GeneratePartition(today) + tomorrowPartition, _ := OneDayPartitionConfiguration.GeneratePartition(tomorrow) + dayAfterTomorrowPartition, _ := OneDayPartitionConfiguration.GeneratePartition(dayAfterTomorrow) + + detachPartitionConfiguration := OneDayPartitionConfiguration + detachPartitionConfiguration.CleanupPolicy = "detach" + + dropPartitionConfiguration := OneDayPartitionConfiguration + OneDayPartitionConfiguration.CleanupPolicy = "drop" + + testCases := []struct { + name string + partitions map[string]postgresql.PartitionConfiguration + existingPartitions []postgresql.Partition + expectedRemovedPartitions []postgresql.Partition + }{ + { + "Drop useless partitions", + map[string]postgresql.PartitionConfiguration{ + "unittest": dropPartitionConfiguration, + }, + []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition, yesterdayPartition, currentPartition, tomorrowPartition, dayAfterTomorrowPartition}, + []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition, dayAfterTomorrowPartition}, + }, + { + "Detach useless partitions", + map[string]postgresql.PartitionConfiguration{ + "unittest": detachPartitionConfiguration, + }, + []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition, yesterdayPartition, currentPartition}, + []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition}, + }, + { + "No cleanup", + map[string]postgresql.PartitionConfiguration{ + "unittest": dropPartitionConfiguration, + }, + []postgresql.Partition{yesterdayPartition, currentPartition, tomorrowPartition}, + []postgresql.Partition{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + logger, postgreSQLMock := getTestMocks(t) // Reset mock on every test case + + for _, partitionConfiguration := range tc.partitions { + table := postgresql.Table{Schema: partitionConfiguration.Schema, Name: partitionConfiguration.Table} + postgreSQLMock.On("ListPartitions", table).Return(tc.existingPartitions, nil).Once() + + for _, partition := range tc.expectedRemovedPartitions { + postgreSQLMock.On("DetachPartition", partition).Return(nil).Once() + + if partitionConfiguration.CleanupPolicy == postgresql.DropCleanupPolicy { + postgreSQLMock.On("DeletePartition", partition).Return(nil).Once() + } + } + } + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, tc.partitions) + err := checker.CleanupPartitions() + + assert.Nil(t, err, "CleanupPartitions should succeed") + postgreSQLMock.AssertExpectations(t) + }) + } +} + +// Test cleanup continue even if a partition could not be dropped or deleted +func TestCleanupPartitionsFailover(t *testing.T) { + successPartitionConfiguration := OneDayPartitionConfiguration + + undetachablePartitionConfiguration := OneDayPartitionConfiguration + undetachablePartitionConfiguration.Table = "undetachable" + + undropablePartitionConfiguration := OneDayPartitionConfiguration + undropablePartitionConfiguration.Table = "undropable" + + configuration := map[string]postgresql.PartitionConfiguration{ + "undetachable": undetachablePartitionConfiguration, + "undropable": undropablePartitionConfiguration, + "success": successPartitionConfiguration, + } + + logger, postgreSQLMock := getTestMocks(t) + + for _, config := range configuration { + table := postgresql.Table{Schema: config.Schema, Name: config.Table} + + dayBeforeYesterdayPartition, _ := config.GeneratePartition(dayBeforeYesterday) + yesterdayPartitionPartition, _ := config.GeneratePartition(yesterday) + todayPartitionPartition, _ := config.GeneratePartition(today) + tomorrowPartitionPartition, _ := config.GeneratePartition(tomorrow) + + partitions := []postgresql.Partition{ + dayBeforeYesterdayPartition, // This partition should be removed by the cleanup + yesterdayPartitionPartition, + todayPartitionPartition, + tomorrowPartitionPartition, + } + + postgreSQLMock.On("ListPartitions", table).Return(partitions, nil).Once() + } + + // Undetachable partition will return an error on detach operation + undetachablePartition, _ := undetachablePartitionConfiguration.GeneratePartition(dayBeforeYesterday) + postgreSQLMock.On("DetachPartition", undetachablePartition).Return(ErrFake).Once() + + // Undropable partition will return an error on drop operation + undropablePartition, _ := undropablePartitionConfiguration.GeneratePartition(dayBeforeYesterday) + postgreSQLMock.On("DetachPartition", undropablePartition).Return(nil).Once() + postgreSQLMock.On("DeletePartition", undropablePartition).Return(ErrFake).Once() + + // Detach and drop will succeed + successPartition, _ := successPartitionConfiguration.GeneratePartition(dayBeforeYesterday) + postgreSQLMock.On("DetachPartition", successPartition).Return(nil).Once() + postgreSQLMock.On("DeletePartition", successPartition).Return(nil).Once() + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, configuration) + err := checker.CleanupPartitions() + + assert.NotNil(t, err, "CleanupPartitions should report an error") + postgreSQLMock.AssertExpectations(t) +} diff --git a/pkg/ppm/mocks/PostgreSQLClient.go b/pkg/ppm/mocks/PostgreSQLClient.go new file mode 100644 index 0000000..99293fc --- /dev/null +++ b/pkg/ppm/mocks/PostgreSQLClient.go @@ -0,0 +1,170 @@ +// Code generated by mockery v2.33.2. DO NOT EDIT. + +package mocks + +import ( + postgresql "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// PostgreSQLClient is an autogenerated mock type for the PostgreSQLClient type +type PostgreSQLClient struct { + mock.Mock +} + +// CreatePartition provides a mock function with given fields: partitionConfiguration, partition +func (_m *PostgreSQLClient) CreatePartition(partitionConfiguration postgresql.PartitionConfiguration, partition postgresql.Partition) error { + ret := _m.Called(partitionConfiguration, partition) + + var r0 error + if rf, ok := ret.Get(0).(func(postgresql.PartitionConfiguration, postgresql.Partition) error); ok { + r0 = rf(partitionConfiguration, partition) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeletePartition provides a mock function with given fields: partition +func (_m *PostgreSQLClient) DeletePartition(partition postgresql.Partition) error { + ret := _m.Called(partition) + + var r0 error + if rf, ok := ret.Get(0).(func(postgresql.Partition) error); ok { + r0 = rf(partition) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DetachPartition provides a mock function with given fields: partition +func (_m *PostgreSQLClient) DetachPartition(partition postgresql.Partition) error { + ret := _m.Called(partition) + + var r0 error + if rf, ok := ret.Get(0).(func(postgresql.Partition) error); ok { + r0 = rf(partition) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetPartitionSettings provides a mock function with given fields: _a0 +func (_m *PostgreSQLClient) GetPartitionSettings(_a0 postgresql.Table) (postgresql.PartitionSettings, error) { + ret := _m.Called(_a0) + + var r0 postgresql.PartitionSettings + var r1 error + if rf, ok := ret.Get(0).(func(postgresql.Table) (postgresql.PartitionSettings, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(postgresql.Table) postgresql.PartitionSettings); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(postgresql.PartitionSettings) + } + + if rf, ok := ret.Get(1).(func(postgresql.Table) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetServerTime provides a mock function with given fields: +func (_m *PostgreSQLClient) GetServerTime() (time.Time, error) { + ret := _m.Called() + + var r0 time.Time + var r1 error + if rf, ok := ret.Get(0).(func() (time.Time, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() time.Time); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Time) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetVersion provides a mock function with given fields: +func (_m *PostgreSQLClient) GetVersion() (int64, error) { + ret := _m.Called() + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func() (int64, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListPartitions provides a mock function with given fields: table +func (_m *PostgreSQLClient) ListPartitions(table postgresql.Table) ([]postgresql.Partition, error) { + ret := _m.Called(table) + + var r0 []postgresql.Partition + var r1 error + if rf, ok := ret.Get(0).(func(postgresql.Table) ([]postgresql.Partition, error)); ok { + return rf(table) + } + if rf, ok := ret.Get(0).(func(postgresql.Table) []postgresql.Partition); ok { + r0 = rf(table) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]postgresql.Partition) + } + } + + if rf, ok := ret.Get(1).(func(postgresql.Table) error); ok { + r1 = rf(table) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewPostgreSQLClient creates a new instance of PostgreSQLClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewPostgreSQLClient(t interface { + mock.TestingT + Cleanup(func()) +}, +) *PostgreSQLClient { + mock := &PostgreSQLClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/ppm/ppm.go b/pkg/ppm/ppm.go new file mode 100644 index 0000000..8800ea4 --- /dev/null +++ b/pkg/ppm/ppm.go @@ -0,0 +1,60 @@ +// Package ppm provides check, provisioning and cleanup methods +package ppm + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +type PostgreSQLClient interface { + CreatePartition(partitionConfiguration postgresql.PartitionConfiguration, partition postgresql.Partition) error + DeletePartition(partition postgresql.Partition) error + DetachPartition(partition postgresql.Partition) error + GetPartitionSettings(postgresql.Table) (postgresql.PartitionSettings, error) + ListPartitions(table postgresql.Table) ([]postgresql.Partition, error) + GetVersion() (int64, error) + GetServerTime() (time.Time, error) +} + +type PPM struct { + ctx context.Context + db PostgreSQLClient + partitions map[string]postgresql.PartitionConfiguration + logger slog.Logger +} + +func New(context context.Context, logger slog.Logger, db PostgreSQLClient, partitions map[string]postgresql.PartitionConfiguration) *PPM { + return &PPM{ + partitions: partitions, + ctx: context, + db: db, + logger: logger, + } +} + +func getExpectedPartitions(partition postgresql.PartitionConfiguration, currentTime time.Time) (partitions []postgresql.Partition, err error) { + retentions, err := partition.GetRetentionPartitions(currentTime) + if err != nil { + return partitions, fmt.Errorf("could not generate retention partitions: %w", err) + } + + current, err := partition.GeneratePartition(currentTime) + if err != nil { + return partitions, fmt.Errorf("could not generate current partition: %w", err) + } + + future, err := partition.GetPreProvisionedPartitions(currentTime) + if err != nil { + return partitions, fmt.Errorf("could not generate preProvisioned partition: %w", err) + } + + partitions = append(partitions, retentions...) + partitions = append(partitions, current) + partitions = append(partitions, future...) + + return +} diff --git a/pkg/ppm/provisioning.go b/pkg/ppm/provisioning.go new file mode 100644 index 0000000..2ee1dad --- /dev/null +++ b/pkg/ppm/provisioning.go @@ -0,0 +1,53 @@ +package ppm + +import ( + "errors" + "fmt" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +var ErrPartitionProvisioningFailed = errors.New("partition provisioning failed for one or more partition") + +func (p PPM) ProvisioningPartitions() error { + currentTime := time.Now() + provisioningFailed := false + + for name, config := range p.partitions { + p.logger.Info("Provisioning partition", "partition", name) + + if err := p.provisionPartitionsFor(config, currentTime); err != nil { + provisioningFailed = true + } + } + + if provisioningFailed { + return ErrPartitionProvisioningFailed + } + + return nil +} + +func (p PPM) provisionPartitionsFor(config postgresql.PartitionConfiguration, at time.Time) error { + provisioningFailed := false + + partitions, err := getExpectedPartitions(config, at) + if err != nil { + return fmt.Errorf("could not generate partition to create: %w", err) + } + + for _, partition := range partitions { + if err := p.db.CreatePartition(config, partition); err != nil { + provisioningFailed = true + + p.logger.Error("Failed to create partition", "error", err, "schema", partition.Schema, "table", partition.Name) + } + } + + if provisioningFailed { + return ErrPartitionProvisioningFailed + } + + return nil +} diff --git a/pkg/ppm/provisioning_test.go b/pkg/ppm/provisioning_test.go new file mode 100644 index 0000000..fd391fb --- /dev/null +++ b/pkg/ppm/provisioning_test.go @@ -0,0 +1,121 @@ +package ppm_test + +import ( + "context" + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/pkg/ppm" + "github.com/stretchr/testify/assert" +) + +func TestProvisioning(t *testing.T) { + dayBeforeYesterdayPartition, _ := OneDayPartitionConfiguration.GeneratePartition(dayBeforeYesterday) + yesterdayPartition, _ := OneDayPartitionConfiguration.GeneratePartition(yesterday) + currentPartition, _ := OneDayPartitionConfiguration.GeneratePartition(today) + tomorrowPartition, _ := OneDayPartitionConfiguration.GeneratePartition(tomorrow) + dayAfterTomorrowPartition, _ := OneDayPartitionConfiguration.GeneratePartition(dayAfterTomorrow) + + TwoDayPartitionConfiguration := OneDayPartitionConfiguration + TwoDayPartitionConfiguration.Retention = 2 + TwoDayPartitionConfiguration.PreProvisioned = 2 + + testCases := []struct { + name string + partitions map[string]postgresql.PartitionConfiguration + expectedCreatedPartitions []postgresql.Partition + }{ + { + "Provisioning create preProvisioned and retention partitions", + map[string]postgresql.PartitionConfiguration{ + "unittest": OneDayPartitionConfiguration, + }, + []postgresql.Partition{ + yesterdayPartition, + currentPartition, + tomorrowPartition, + }, + }, + { + "Multiple provisioning", + map[string]postgresql.PartitionConfiguration{ + "unittest 1": TwoDayPartitionConfiguration, + "unittest 2": TwoDayPartitionConfiguration, + }, + []postgresql.Partition{ + dayBeforeYesterdayPartition, + yesterdayPartition, + currentPartition, + tomorrowPartition, + dayAfterTomorrowPartition, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + logger, postgreSQLMock := getTestMocks(t) // Reset mock on every test case + + for _, partitionConfiguration := range tc.partitions { + for _, partition := range tc.expectedCreatedPartitions { + postgreSQLMock.On("CreatePartition", partitionConfiguration, partition).Return(nil).Once() + } + } + + provisionner := ppm.New(context.TODO(), *logger, postgreSQLMock, tc.partitions) + err := provisionner.ProvisioningPartitions() + + assert.Nil(t, err, "ProvisioningPartitions should succeed") + postgreSQLMock.AssertExpectations(t) + }) + } +} + +// Test provisioning continue even if a partition could not be created +func TestProvisioningFailover(t *testing.T) { + failedPartitionConfiguration := OneDayPartitionConfiguration + failedPartitionConfiguration.Table = "failed" + + testCases := []struct { + name string + config postgresql.PartitionConfiguration + createPartitionError error + }{ + { + "failed provisioning", + failedPartitionConfiguration, + ErrFake, + }, + { + "success provisioning", + OneDayPartitionConfiguration, + nil, + }, + } + + logger, postgreSQLMock := getTestMocks(t) + + configuration := map[string]postgresql.PartitionConfiguration{} + + for _, tc := range testCases { + previous, _ := tc.config.GetRetentionPartitions(today) + current, _ := tc.config.GeneratePartition(today) + future, _ := tc.config.GetPreProvisionedPartitions(today) + + partitions := []postgresql.Partition{current} + partitions = append(partitions, previous...) + partitions = append(partitions, future...) + + for _, partition := range partitions { + postgreSQLMock.On("CreatePartition", tc.config, partition).Return(tc.createPartitionError).Once() + } + + configuration[tc.name] = tc.config + } + + provisionner := ppm.New(context.TODO(), *logger, postgreSQLMock, configuration) + err := provisionner.ProvisioningPartitions() + + assert.NotNil(t, err, "ProvisioningPartitions should report an error") + postgreSQLMock.AssertExpectations(t) +} diff --git a/pkg/ppm/server.go b/pkg/ppm/server.go new file mode 100644 index 0000000..1cc4222 --- /dev/null +++ b/pkg/ppm/server.go @@ -0,0 +1,80 @@ +package ppm + +import ( + "errors" + "fmt" + "time" +) + +const ( + postgreSQLMinimalVersion = 14 // Minimal supported PostgreSQL version + timeDriftTolerance = 10 * time.Second // Maximum time drift between client and server +) + +var ( + ErrUnsupportedServer = errors.New("unsupported PostgreSQL version") + ErrTimeDrift = errors.New("client and server time drift") +) + +func (p *PPM) CheckServerRequirements() error { + if err := p.requirePostgreSQLSupportedVersion(); err != nil { + return fmt.Errorf("error checking PostgreSQL version: %w", err) + } + + if err := p.requireNoTimeDrift(); err != nil { + return fmt.Errorf("error checking time drift: %w", err) + } + + return nil +} + +func (p *PPM) requirePostgreSQLSupportedVersion() error { + version, err := p.db.GetVersion() + if err != nil { + return fmt.Errorf("failed to fetch PostgreSQL version: %w", err) + } + + if version < postgreSQLMinimalVersion { + p.logger.Error("Unsupported PostgreSQL version", "current_version", version, "minimal_version", postgreSQLMinimalVersion) + + return ErrUnsupportedServer + } + + return nil +} + +func (p *PPM) requireNoTimeDrift() error { + isSync, err := p.clientAndServerAreTimeSynchronized(timeDriftTolerance) + if err != nil { + p.logger.Error("Error checking time synchronization", "error", err) + + return fmt.Errorf("error checking time synchronization: %w", err) + } + + if !isSync { + p.logger.Error("Client and server times are not synchronized within the tolerance.", "time_tolerance", timeDriftTolerance) + + return ErrTimeDrift + } + + return nil +} + +func (p *PPM) clientAndServerAreTimeSynchronized(tolerance time.Duration) (bool, error) { + serverTime, err := p.db.GetServerTime() + if err != nil { + return false, fmt.Errorf("failed to get server time: %w", err) + } + + clientTime := time.Now().UTC() + diff := clientTime.Sub(serverTime) + + // Check if the absolute difference is within the tolerance + if diff < 0 { + diff = -diff + } + + isSync := diff <= tolerance + + return isSync, nil +} diff --git a/pkg/ppm/server_test.go b/pkg/ppm/server_test.go new file mode 100644 index 0000000..ca64807 --- /dev/null +++ b/pkg/ppm/server_test.go @@ -0,0 +1,66 @@ +package ppm_test + +import ( + "context" + "testing" + "time" + + "github.com/qonto/postgresql-partition-manager/pkg/ppm" + "github.com/stretchr/testify/assert" +) + +func TestServerRequirements(t *testing.T) { + shouldSucceed := true + shouldFail := false + + testCases := []struct { + name string + serverVersion int64 + serverTime time.Time + expected bool + }{ + { + "Synchronized PostgreSQL 14", + 14, + time.Now(), + shouldSucceed, + }, + { + "Unsupported PostgreSQL 13", + 13, + time.Now(), + shouldFail, + }, + { + "PostgreSQL 14 server in the future", + 14, + time.Now().Add(time.Second * 30), + shouldFail, + }, + { + "PostgreSQL 14 server in the past", + 14, + time.Now().Add(time.Second * -30), + shouldFail, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Reset mock on every test case + logger, postgreSQLMock := getTestMocks(t) + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, nil) + + postgreSQLMock.On("GetVersion").Return(tc.serverVersion, nil).Once() + postgreSQLMock.On("GetServerTime").Return(tc.serverTime, nil).Once() + + err := checker.CheckServerRequirements() + + if tc.expected == shouldSucceed { + assert.Nil(t, err, "ServerRequirement should match") + } else { + assert.NotNil(t, err, "error checking time drift") + } + }) + } +} From 8478b1ec69b7b795dba7fd462c8ede51df6baacf Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:18:28 +0200 Subject: [PATCH 11/27] chore(go): Add CLI commands --- cmd/root.go | 126 ++++++++++++++++++++++++++ cmd/run/run.go | 156 ++++++++++++++++++++++++++++++++ cmd/validate/validate.go | 48 ++++++++++ internal/infra/config/config.go | 56 ++++++++++++ main.go | 7 ++ 5 files changed, 393 insertions(+) create mode 100644 cmd/root.go create mode 100644 cmd/run/run.go create mode 100644 cmd/validate/validate.go create mode 100644 internal/infra/config/config.go create mode 100644 main.go diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..f7dac06 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,126 @@ +// Package cmd implements command to start the PostgreSQL Partition Manager +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/qonto/postgresql-partition-manager/cmd/run" + "github.com/qonto/postgresql-partition-manager/cmd/validate" + "github.com/qonto/postgresql-partition-manager/internal/infra/build" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + configErrorExitCode = 1 + cmdErrorExitCode = 3 +) + +var ( + cfgFile string + logFormat string + debug bool + connectionURL string + lockTimeout string + statementTimeout string +) + +func NewRootCommand() (*cobra.Command, error) { + cmd := &cobra.Command{ + Use: "postgresql-partition-manager ", + Version: fmt.Sprintf("%s, commit %s, built at %s", build.Version, build.CommitSHA, build.Date), + Short: "PostgreSQL partition manager", + Long: "Simplified PostgreSQL partioning management", + TraverseChildren: true, + } + + cobra.OnInitialize(initConfig) + + cmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is $HOME/postgresql-partition-manager.yaml)") + cmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Enable debug mode") + cmd.PersistentFlags().StringVarP(&logFormat, "log-format", "l", "json", "Log format (text or json)") + cmd.PersistentFlags().StringVarP(&connectionURL, "connection-url", "u", "", "Database connection string") + cmd.PersistentFlags().StringVarP(&lockTimeout, "lock-timeout", "", "100", "Set lock_timeout (ms)") + cmd.PersistentFlags().StringVarP(&statementTimeout, "statement-timeout", "", "3000", "Set statement_timeout (ms)") + + err := viper.BindPFlag("debug", cmd.PersistentFlags().Lookup("debug")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'debug' parameter: %w", err) + } + + err = viper.BindPFlag("log-format", cmd.PersistentFlags().Lookup("log-format")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'log-format' parameter: %w", err) + } + + err = viper.BindPFlag("connection-url", cmd.PersistentFlags().Lookup("connection-url")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'connection-url' parameter: %w", err) + } + + err = viper.BindPFlag("lock-timeout", cmd.PersistentFlags().Lookup("lock-timeout")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'lock-timeout' parameter: %w", err) + } + + err = viper.BindPFlag("statement-timeout", cmd.PersistentFlags().Lookup("statement-timeout")) + if err != nil { + return cmd, fmt.Errorf("failed to bind 'statement-timeout' parameter: %w", err) + } + + return cmd, nil +} + +func Execute() { + cmd, err := NewRootCommand() + if err != nil { + fmt.Println("ERROR: Failed to load configuration: %w", err) + os.Exit(configErrorExitCode) + } + + cmd.AddCommand(validate.ValidateCmd()) + cmd.AddCommand(run.RunCmd()) + + err = cmd.Execute() + if err != nil { + fmt.Println("ERROR: Command failed: %w", err) + os.Exit(cmdErrorExitCode) + } +} + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory + home, err := os.UserHomeDir() + cobra.CheckErr(err) + + // Search config in home directory or current directory with name "postgresql-partition-manager.yaml" + + configurationFilename := "postgresql-partition-manager.yaml" + currentPathFilename := configurationFilename + homeFilename := filepath.Join(home, configurationFilename) + + if _, err := os.Stat(homeFilename); err == nil { + viper.SetConfigFile(homeFilename) + } + + if _, err := os.Stat(currentPathFilename); err == nil { + viper.SetConfigFile(currentPathFilename) + } + } + + if err := viper.ReadInConfig(); err == nil { + fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + } + + viper.SetEnvPrefix("postgresql_partition_manager") // will be uppercased automatically + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() +} diff --git a/cmd/run/run.go b/cmd/run/run.go new file mode 100644 index 0000000..2eef93d --- /dev/null +++ b/cmd/run/run.go @@ -0,0 +1,156 @@ +// Package run provides Cobra commands to execute PPM +package run + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + + "github.com/qonto/postgresql-partition-manager/internal/infra/config" + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/pkg/ppm" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + InvalidConfigurationExitCode = 1 + InternalErrorExitCode = 2 + DatabaseErrorExitCode = 3 + PartitionsProvisioningFailedExitCode = 4 + PartitionsCheckFailedExitCode = 5 + PartitionsCleanupFailedExitCode = 6 +) + +var ErrUnsupportedPostgreSQLVersion = errors.New("unsupported PostgreSQL version") + +func RunCmd() *cobra.Command { + runCmd := &cobra.Command{ + Use: "run", + Short: "Perform partition operations", + Long: "Perform partition operations", + Run: func(cmd *cobra.Command, args []string) { + _ = cmd.Help() + }, + } + + runCmd.AddCommand(AllCmd) + runCmd.AddCommand(CheckCmd) + runCmd.AddCommand(ProvisioningCmd) + runCmd.AddCommand(CleanupCmd) + + return runCmd +} + +var AllCmd = &cobra.Command{ + Use: "all", + Short: "Perform partitions provisioning, cleanup, and check", + Long: "Perform partitions provisioning, cleanup, and check", + Run: func(cmd *cobra.Command, args []string) { + client, logger := initCmd() + + provisioningCmd(client, logger) + cleanupCmd(client, logger) + checkCmd(client, logger) + }, +} + +var CheckCmd = &cobra.Command{ + Use: "check", + Short: "Check existing partitions", + Long: "Check existing partitions", + Run: func(cmd *cobra.Command, args []string) { + client, logger := initCmd() + checkCmd(client, logger) + }, +} + +var CleanupCmd = &cobra.Command{ + Use: "cleanup", + Short: "Remove outdated partitions", + Long: "Remove outdated partitions", + Run: func(cmd *cobra.Command, args []string) { + client, logger := initCmd() + cleanupCmd(client, logger) + }, +} + +var ProvisioningCmd = &cobra.Command{ + Use: "provisioning", + Short: "Create and attach new partitions", + Long: "Create and attach new partitions", + Run: func(cmd *cobra.Command, args []string) { + client, logger := initCmd() + provisioningCmd(client, logger) + }, +} + +func initCmd() (*ppm.PPM, *slog.Logger) { + var config config.Config + + if err := viper.Unmarshal(&config); err != nil { + fmt.Println("ERROR: Unable to load configuration", "error", err) + os.Exit(InvalidConfigurationExitCode) + } + + err := config.Check() + if err != nil { + os.Exit(InvalidConfigurationExitCode) + } + + log, err := logger.New(config.Debug, config.LogFormat) + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + os.Exit(InternalErrorExitCode) + } + + databaseConfiguration := postgresql.ConnectionSettings{ + URL: config.ConnectionURL, + LockTimeout: config.LockTimeout, + StatementTimeout: config.StatementTimeout, + } + + conn, err := postgresql.GetDatabaseConnection(databaseConfiguration) + if err != nil { + log.Error("Could not connect to database", "error", err) + os.Exit(DatabaseErrorExitCode) + } + + db := postgresql.New(*log, conn) + + client := ppm.New(context.TODO(), *log, db, config.Partitions) + + if err = client.CheckServerRequirements(); err != nil { + log.Error("Server is incompatible", "error", err) + os.Exit(DatabaseErrorExitCode) + } + + return client, log +} + +func checkCmd(client *ppm.PPM, logger *slog.Logger) { + if err := client.CheckPartitions(); err != nil { + os.Exit(PartitionsCheckFailedExitCode) + } + + logger.Info("All partitions are correctly configured") +} + +func cleanupCmd(client *ppm.PPM, logger *slog.Logger) { + if err := client.CleanupPartitions(); err != nil { + os.Exit(PartitionsCleanupFailedExitCode) + } + + logger.Info("All partitions are cleaned") +} + +func provisioningCmd(client *ppm.PPM, logger *slog.Logger) { + if err := client.ProvisioningPartitions(); err != nil { + os.Exit(PartitionsProvisioningFailedExitCode) + } + + logger.Info("All partitions are correctly provisioned") +} diff --git a/cmd/validate/validate.go b/cmd/validate/validate.go new file mode 100644 index 0000000..5ffca47 --- /dev/null +++ b/cmd/validate/validate.go @@ -0,0 +1,48 @@ +// Package validate provides Cobra command to validate the configuration file +package validate + +import ( + "fmt" + "os" + + "github.com/qonto/postgresql-partition-manager/internal/infra/config" + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + InvalidConfigurationExitCode = 1 + InternalErrorExitCode = 2 +) + +func ValidateCmd() *cobra.Command { + validateCmd := &cobra.Command{ + Use: "validate", + Short: "Check configuration file", + Long: `Check configuration file and exit with an error if configuration is invalid`, + TraverseChildren: true, + Run: func(cmd *cobra.Command, args []string) { + var config config.Config + + if err := viper.Unmarshal(&config); err != nil { + fmt.Printf("Unable to load configuration, %v", err) + os.Exit(InvalidConfigurationExitCode) + } + + logger, err := logger.New(config.Debug, config.LogFormat) + if err != nil { + fmt.Println("ERROR: Fail to initialize logger: %w", err) + os.Exit(InternalErrorExitCode) + } + + if err := config.Check(); err != nil { + os.Exit(InvalidConfigurationExitCode) + } + + logger.Info("Configuration is valid") + }, + } + + return validateCmd +} diff --git a/internal/infra/config/config.go b/internal/infra/config/config.go new file mode 100644 index 0000000..73f8ff7 --- /dev/null +++ b/internal/infra/config/config.go @@ -0,0 +1,56 @@ +// Package config provides PPM configuration settings +package config + +import ( + "errors" + "fmt" + + "github.com/go-playground/validator/v10" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +type Config struct { + Debug bool `mapstructure:"debug"` + LogFormat string `mapstructure:"log-format"` + ConnectionURL string `mapstructure:"connection-url"` + StatementTimeout int `mapstructure:"statement-timeout" validate:"required"` + LockTimeout int `mapstructure:"lock-timeout" validate:"required"` + Partitions map[string]postgresql.PartitionConfiguration `mapstructure:"partitions" validate:"required,dive,keys,endkeys,required"` +} + +func (c *Config) Check() error { + validate := validator.New() + + err := validate.Struct(c) + if err != nil { + formatConfigurationError(err) + + return fmt.Errorf("configuration validation failed: %w", err) + } + + return nil +} + +func formatConfigurationError(err error) { + var invalidValidation *validator.InvalidValidationError + + if errors.As(err, &invalidValidation) { + fmt.Println("ERROR: The provided configuration is invalid and cannot be processed. Please check your configuration for any structural errors.") + } + + var validationErrors validator.ValidationErrors + + if errors.As(err, &validationErrors) { + for _, e := range validationErrors { + switch e.Tag() { + case "required": + fmt.Printf("ERROR: The '%s' field is required and cannot be empty.\n", e.StructNamespace()) + case "oneof": + fmt.Printf("ERROR: The '%s' field must be one of [%s], but got '%s'.\n", e.StructNamespace(), e.Param(), e.Value()) + default: + // Generic error message for any other validation tags + fmt.Printf("ERROR: The '%s' field is not valid: %s\n", e.Field(), e.Error()) + } + } + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..a11021b --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/qonto/postgresql-partition-manager/cmd" + +func main() { + cmd.Execute() +} From 6b902152cec7787d8d1dad2899f4387e867bc8fb Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:16:51 +0200 Subject: [PATCH 12/27] chore(bats): Add bats tests --- CONTRIBUTING.md | 300 ++++++++++++++++++ scripts/bats/00_cli.bats | 18 ++ scripts/bats/10_validate.bats | 23 ++ scripts/bats/20_check.bats | 108 +++++++ scripts/bats/30_provisioning.bats | 89 ++++++ scripts/bats/40_cleanup.bats | 62 ++++ scripts/bats/configuration/template.yaml | 6 + scripts/bats/configuration/valid.yaml | 16 + scripts/bats/test/libs/dependencies.bash | 12 + scripts/bats/test/libs/partitions.bash | 53 ++++ scripts/bats/test/libs/seeds.bash | 40 +++ scripts/bats/test/libs/sql.bash | 55 ++++ .../localdev/configuration/bats/Dockerfile | 4 + 13 files changed, 786 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 scripts/bats/00_cli.bats create mode 100644 scripts/bats/10_validate.bats create mode 100644 scripts/bats/20_check.bats create mode 100644 scripts/bats/30_provisioning.bats create mode 100644 scripts/bats/40_cleanup.bats create mode 100644 scripts/bats/configuration/template.yaml create mode 100644 scripts/bats/configuration/valid.yaml create mode 100644 scripts/bats/test/libs/dependencies.bash create mode 100644 scripts/bats/test/libs/partitions.bash create mode 100644 scripts/bats/test/libs/seeds.bash create mode 100644 scripts/bats/test/libs/sql.bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e8aa025 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,300 @@ +# Contributing + +PostgreSQL Partition Manager uses GitHub to manage reviews of pull requests. + +* If you are a new contributor, see: [Steps to Contribute](#steps-to-contribute) + +* If you have a trivial fix or improvement, go ahead and create a pull request + +* Relevant coding style guidelines are the [Go Code Review + Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) + and the _Formatting and style_ section of Peter Bourgon's [Go: Best + Practices for Production + Environments](https://peter.bourgon.org/go-in-production/#formatting-and-style). + +* Be sure to enable [signed commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) + +## Steps to Contribute + +Should you wish to work on an issue, please claim it first by commenting on the GitHub issue that you want to work on it. This is to prevent duplicated efforts from contributors on the same issue. + +All our issues are regularly tagged so you can filter down the issues involving the components you want to work on. + +For quickly compiling and testing your changes do: + +```bash +# For building. +make build +./postgresql-partition-manager + +# For testing. +make test # Make sure all the tests pass before you commit and push :) +``` + +We use: + +* [`pre-commit`](https://pre-commit.com) to make right first time changes. Enable it for this repository with `pre-commit install`. + +* [`golangci-lint`](https://github.com/golangci/golangci-lint) for linting the code. If it reports an issue and you think that the warning needs to be disregarded or is a false-positive, you can add a special comment `//nolint:linter1[,linter2,...]` before the offending line. Use this sparingly though, fixing the code to comply with the linter's recommendation is in general the preferred course of action. + +* [`markdownlint-cli2`](https://github.com/DavidAnson/markdownlint-cli2) for linting the Markdown documents. + +* [`yamllint`](https://github.com/adrienverge/yamllint) for linting the YAML documents. + +## Pull Request Checklist + +* Branch from the `main` branch and, if needed, rebase to the current main branch before submitting your pull request. If it doesn't merge cleanly with main you may be asked to rebase your changes. + +* Commits should be as small as possible while ensuring each commit is correct independently (i.e., each commit should compile and pass tests). + +* Add tests relevant to the fixed bug or new feature. + +## Install pre-commit + +1. Install [pre-commit](https://pre-commit.com/) + +1. Install [markdownlint-cli2](https://github.com/DavidAnson/markdownlint-cli2) + +1. Enable pre-commit for the repository + + ```bash + pre-commit install + ``` + +## Local development + + + +
+Docker + +1. Install requirements + + * [Golang 1.21](https://go.dev/doc/install) + + Optionals: + + * [Orbstack](https://orbstack.dev/) (recommended) or Docker + +1. Setup PostgreSQL + + Via docker containers: + + ```bash + cd scripts/localdev/ + export POSTGRESQL_VERSION=16 # Optional. Override PostgreSQL version + docker compose up -d postgres + ``` + + Or manually: + + ```sql + \i scripts/localdev/configuration/postgresql/seeds/00_database.sql + \i scripts/localdev/configuration/postgresql/seeds/10_by_date.sql + \i scripts/localdev/configuration/postgresql/seeds/10_by_timestamp.sql + \i scripts/localdev/configuration/postgresql/seeds/10_by_uuidv7.sql + ``` + +1. Build application from the root directory + + ```bash + make build + ``` + +1. Optional. Create configuration file + + ```bash + cat > postgresql-partition-manager.yaml << EOF + --- + debug: true + + log-format: text + + connection-url: postgres://postgres:hackme@localhost/partitions + + partitions: + by_date: + schema: public + table: by_date + partitionKey: created_at + interval: yearly + retention: 7 + preProvisioned: 7 + cleanupPolicy: drop + by_timestamp: + schema: public + table: by_timestamp + partitionKey: created_at + interval: daily + retention: 7 + preProvisioned: 7 + cleanupPolicy: drop + by_uuidv7: + schema: public + table: by_uuidv7 + partitionKey: id + interval: monthly + retention: 3 + preProvisioned: 1 + cleanupPolicy: drop + EOF + ``` + + Run provisioning script to perform provisioning, clean up, and check operations + + ```bash + ./postgresql-partition-manager run all + ``` + +
+ +
+Kubernetes + +The Kubernetes local development environment located in `scripts/kubernetesdev` is designed to facilitate Helm chart development and QA in containerized environment. + +Requirements: + +* Orbstack, with [Kubernetes environment enabled](https://docs.orbstack.dev/kubernetes/) + +Steps: + +1. Build application (from repository root directory) + + ```bash + docker build . -t postgresql-partition-manager:dev + ``` + +1. Build Helm chart dependencies + + ```bash + cd scripts/kubernetesdev/ + helm dependency build --skip-refresh + ``` + +1. Set deployment parameters + + ```bash + KUBERNETES_NAMESPACE=default # Replace with your namespace + HELM_RELEASE_NAME=main # Replace with an helm release + ``` + +1. Trigger PostgreSQL and Postgresql Partition Manager deployments + + Optional. Adjust deployment settings in `values.yaml`. + + ```bash + helm upgrade ${HELM_RELEASE_NAME} . --install --values values.yaml + ``` + +1. Trigger the PostgreSQL Partition Manager job manually + + Set a Kubernetes job name: + + ```bash + MANUAL_JOB=ppm-manually-triggered + ``` + + Trigger job manually: + + ```bash + kubectl create job --namespace ${KUBERNETES_NAMESPACE} --from=cronjob/${HELM_RELEASE_NAME}-postgresql-partition-manager ${MANUAL_JOB} + ``` + + Check cronjob execution: + + ```bash + kubectl describe job --namespace ${KUBERNETES_NAMESPACE} ${MANUAL_JOB} + ``` + + Check application logs + + ```bash + # PostgreSQL logs + kubectl logs --namespace ${KUBERNETES_NAMESPACE} deployments/postgres + + # PostgreSQL partition manager + kubectl logs --namespace ${KUBERNETES_NAMESPACE} --selector=job-name=${MANUAL_JOB} + ``` + + Clean up manual job + + ```bash + kubectl delete job --namespace ${KUBERNETES_NAMESPACE} ${MANUAL_JOB} + ``` + +1. Cleanup, delete PostgreSQL deployment + + ```bash + helm uninstall ${HELM_RELEASE_NAME} + ``` + +Useful commands: + +Connect to PostgreSQL + +```bash +export PGHOST=localhost +export PGPORT=$(kubectl get svc postgres -o jsonpath='{.spec.ports[0].nodePort}') +export PGUSER=$(kubectl get secret postgres-credentials --template={{.data.user}} | base64 -D) +export PGPASSWORD=$(kubectl get secret postgres-credentials --template={{.data.password}} | base64 -D) +export PGDATABASE=$(kubectl get configmap postgres-configuration --template={{.data.database}}) +psql +``` + +
+ +## Tests + +### Bats + +1. Install dependencies + + ```bash + brew install bats-core + brew tap bats-core/bats-core + brew install bats-support + brew install bats-assert + brew install yq + brew install libpq # or postgresql + ``` + +1. Start PostgreSQL + +1. Export environment variables + + ```bash + export PGHOST=localhost + export PGDATABASE=unittest + export PGUSER=postgres + export PGPASSWORD=hackme + ``` + +1. Launch tests + + ```bash + make bats-test + ``` + +## Dependency management + +Project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. + +To add or update a new dependency, use the `go get` command: + +```bash +# Pick the latest tagged release. +go get example.com/some/module/pkg@latest + +# Pick a specific version. +go get example.com/some/module/pkg@vX.Y.Z +``` + +Tidy up the `go.mod` and `go.sum` files: + +```bash +# The GO111MODULE variable can be omitted when the code isn't located in GOPATH. +GO111MODULE=on go mod tidy +``` + +You have to commit the changes to `go.mod` and `go.sum` before submitting the pull request. diff --git a/scripts/bats/00_cli.bats b/scripts/bats/00_cli.bats new file mode 100644 index 0000000..779a237 --- /dev/null +++ b/scripts/bats/00_cli.bats @@ -0,0 +1,18 @@ +setup() { + bats_load_library bats-support + bats_load_library bats-assert +} + +@test "Test returns its version" { + run postgresql-partition-manager --version + + assert_success + assert_output --partial "postgresql-partition-manager version development" +} + +@test "Test help message" { + run postgresql-partition-manager --help + + assert_success + assert_output --partial "Simplified PostgreSQL partioning management" +} diff --git a/scripts/bats/10_validate.bats b/scripts/bats/10_validate.bats new file mode 100644 index 0000000..e030750 --- /dev/null +++ b/scripts/bats/10_validate.bats @@ -0,0 +1,23 @@ +setup() { + bats_load_library bats-support + bats_load_library bats-assert +} + +# Test for program's behavior when the configuration file is empty +@test "Program should exit when passed an empty configuration file" { + run postgresql-partition-manager validate -c /dev/null + + assert_failure + assert_equal "$status" 1 + + # Verify that mandatory fields are reported as errors + assert_line "ERROR: The 'Config.ConnectionURL' field is required and cannot be empty." + assert_line "ERROR: The 'Config.Partitions' field is required and cannot be empty." +} + +@test "Ensure validate command executes successfully with a valid configuration file" { + run postgresql-partition-manager validate -c configuration/valid.yaml + + assert_success + assert_output --partial "Configuration is valid" +} diff --git a/scripts/bats/20_check.bats b/scripts/bats/20_check.bats new file mode 100644 index 0000000..b9be9ac --- /dev/null +++ b/scripts/bats/20_check.bats @@ -0,0 +1,108 @@ +load 'test/libs/dependencies' +load 'test/libs/partitions' +load 'test/libs/seeds' +load 'test/libs/sql' + +setup() { + bats_load_library bats-support + bats_load_library bats-assert + + reset_database +} + +@test "Test exit code on PostgreSQL connection error" { + run postgresql-partition-manager run check -c configuration/valid.yaml --connection-url an-invalid-connection-url + + assert_failure + assert_equal "$status" 3 + assert_output --partial "Could not connect to database" +} + +@test "Test check return a success on valid configuration" { + local TABLE=$(generate_table_name) + local INTERVAL=daily + local RETENTION=2 + local PREPROVISIONED=2 + + # Create partioned table 2 retention days + create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED} + + local CONFIGURATION=$(cat << EOF +partitions: + unittest: + schema: public + table: ${TABLE} + interval: ${INTERVAL} + partitionKey: created_at + cleanupPolicy: drop + retention: ${RETENTION} + preProvisioned: ${PREPROVISIONED} +EOF +) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + run postgresql-partition-manager run check -c ${CONFIGURATION_FILE} + + assert_success + assert_output --partial "All partitions are correctly configured" +} + +@test "Test check return an error when retention partitions are missing" { + local TABLE=$(generate_table_name) + local INTERVAL=daily + local INITIAL_RETENTION=1 + local NEW_RETENTION=2 + local PREPROVISIONED=1 + + create_partioned_table ${TABLE} ${TABLE} ${INTERVAL} ${INITIAL_RETENTION} ${PREPROVISIONED} + + # Generate configuration with only 1 retention + local CONFIGURATION=$(cat << EOF +partitions: + unittest: + schema: public + table: ${TABLE} + interval: daily + partitionKey: created_at + cleanupPolicy: drop + retention: ${NEW_RETENTION} # This should generate an error + preProvisioned: ${PREPROVISIONED} +EOF +) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + run postgresql-partition-manager run check -c ${CONFIGURATION_FILE} + + assert_failure + assert_output --partial "Found missing tables" +} + +@test "Test check return an error when preProvisioned partitions are missing" { + local TABLE=$(generate_table_name) + local INTERVAL=daily + local RETENTION=2 + local INITIAL_PREPROVISIONED=2 + local NEW_PREPROVISIONED=3 + + create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${INITIAL_PREPROVISIONED} + + # Increase preProvisioned partitions + local CONFIGURATION=$(cat << EOF +partitions: + unittest: + schema: public + table: ${TABLE} + interval: daily + partitionKey: created_at + cleanupPolicy: drop + retention: ${RETENTION} + preProvisioned: ${NEW_PREPROVISIONED} # This will generate an error +EOF +) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + run postgresql-partition-manager run check -c ${CONFIGURATION_FILE} + + assert_failure + assert_output --partial "Found missing tables" +} diff --git a/scripts/bats/30_provisioning.bats b/scripts/bats/30_provisioning.bats new file mode 100644 index 0000000..dfea682 --- /dev/null +++ b/scripts/bats/30_provisioning.bats @@ -0,0 +1,89 @@ +load 'test/libs/dependencies' +load 'test/libs/partitions' +load 'test/libs/seeds' +load 'test/libs/sql' + +setup() { + bats_load_library bats-support + bats_load_library bats-assert + + reset_database +} + +@test "Test that provisioning succeed on up-to-date partitioning" { + local TABLE=$(generate_table_name) + local INTERVAL=daily + local RETENTION=1 + local PREPROVISIONED=1 + + # Create partioned table + create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED} + + local CONFIGURATION=$(cat << EOF +partitions: + unittest: + schema: public + table: ${TABLE} + interval: ${INTERVAL} + partitionKey: created_at + cleanupPolicy: drop + retention: ${RETENTION} + preProvisioned: ${PREPROVISIONED} +EOF +) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE} + + assert_success + assert_output --partial "All partitions are correctly provisioned" + assert_table_exists public $(generate_daily_partition_name ${TABLE} -1) # retention partition + assert_table_exists public $(generate_daily_partition_name ${TABLE} 0) # current partition + assert_table_exists public $(generate_daily_partition_name ${TABLE} 1) # preProvisioned partition +} + +@test "Test that preProvisioned and retention partitions can be increased" { + local TABLE=$(generate_table_name) + local INTERVAL=daily + local RETENTION=1 + local PREPROVISIONED=1 + + # Create partioned table + create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED} + + local CONFIGURATION=$(cat << EOF +partitions: + unittest: + schema: public + table: ${TABLE} + interval: ${INTERVAL} + partitionKey: created_at + cleanupPolicy: drop + retention: ${RETENTION} + preProvisioned: ${PREPROVISIONED} +EOF +) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE} + + + assert_success + assert_output --partial "All partitions are correctly provisioned" + assert_table_exists public $(generate_daily_partition_name ${TABLE} -1) # retention partition + assert_table_exists public $(generate_daily_partition_name ${TABLE} 0) # current partition + assert_table_exists public $(generate_daily_partition_name ${TABLE} 1) # preProvisioned partition + + # Increase retention and preProvisioned partitions + local NEW_RETENTION=2 + local NEW_PREPROVISIONED=3 + yq eval ".partitions.unittest.retention = ${NEW_RETENTION}" -i ${CONFIGURATION_FILE} + yq eval ".partitions.unittest.preProvisioned = ${NEW_PREPROVISIONED}" -i ${CONFIGURATION_FILE} + + run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE} + + assert_success + assert_output --partial "All partitions are correctly provisioned" + assert_table_exists public $(generate_daily_partition_name ${TABLE} -${NEW_RETENTION}) # New retention partition + assert_table_exists public $(generate_daily_partition_name ${TABLE} ${NEW_PREPROVISIONED}) # New preProvisioned partition +} diff --git a/scripts/bats/40_cleanup.bats b/scripts/bats/40_cleanup.bats new file mode 100644 index 0000000..fd4d803 --- /dev/null +++ b/scripts/bats/40_cleanup.bats @@ -0,0 +1,62 @@ +load 'test/libs/dependencies' +load 'test/libs/partitions' +load 'test/libs/seeds' +load 'test/libs/sql' + +setup() { + bats_load_library bats-support + bats_load_library bats-assert + + reset_database +} + +@test "Test that useless partitions are removed by the cleanup" { + local TABLE=$(generate_table_name) + local INTERVAL=daily + local INITIAL_RETENTION=3 + local INITIAL_PREPROVISIONED=3 + + # Create partioned table + create_partioned_table ${TABLE} ${INTERVAL} ${INITIAL_RETENTION} ${INITIAL_PREPROVISIONED} + + for ((i=1; i<= INITIAL_RETENTION; i++));do + assert_table_exists public $(generate_daily_partition_name ${TABLE} -${i}) + done + + for ((i=1; i<= INITIAL_PREPROVISIONED; i++));do + assert_table_exists public $(generate_daily_partition_name ${TABLE} ${i}) + done + + # Set lower retention and preProvisioned + local NEW_RETENTION=1 + local NEW_PREPROVISIONED=1 + local CONFIGURATION=$(cat << EOF +partitions: + unittest: + schema: public + table: ${TABLE} + interval: ${INTERVAL} + partitionKey: created_at + cleanupPolicy: drop + retention: ${NEW_RETENTION} + preProvisioned: ${NEW_PREPROVISIONED} +EOF +) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + run postgresql-partition-manager run cleanup -c ${CONFIGURATION_FILE} + + assert_success + assert_output --partial "All partitions are cleaned" + assert_table_exists public $(generate_daily_partition_name ${TABLE} 1) # Should exist + assert_table_exists public $(generate_daily_partition_name ${TABLE} 0) # current partition must still exists + assert_table_exists public $(generate_daily_partition_name ${TABLE} -1) # Should exist + + for ((i=NEW_RETENTION +1; i<= INITIAL_RETENTION; i++));do + assert_table_not_exists public $(generate_daily_partition_name ${TABLE} -${i}) + done + + for ((i=NEW_RETENTION +1; i<= INITIAL_PREPROVISIONED; i++));do + assert_table_not_exists public $(generate_daily_partition_name ${TABLE} ${i}) + done +} diff --git a/scripts/bats/configuration/template.yaml b/scripts/bats/configuration/template.yaml new file mode 100644 index 0000000..5339d2f --- /dev/null +++ b/scripts/bats/configuration/template.yaml @@ -0,0 +1,6 @@ +--- +debug: true + +log-format: text + +connection-url: postgres://postgres:hackme@localhost/unittest diff --git a/scripts/bats/configuration/valid.yaml b/scripts/bats/configuration/valid.yaml new file mode 100644 index 0000000..1465ebe --- /dev/null +++ b/scripts/bats/configuration/valid.yaml @@ -0,0 +1,16 @@ +--- +debug: true + +log-format: text + +connection-url: postgres://postgres:hackme@localhost/unittest + +partitions: + by_date: + schema: public + table: by_date + partitionKey: created_at + interval: yearly + retention: 1 + preProvisioned: 1 + cleanupPolicy: drop diff --git a/scripts/bats/test/libs/dependencies.bash b/scripts/bats/test/libs/dependencies.bash new file mode 100644 index 0000000..de8cdd5 --- /dev/null +++ b/scripts/bats/test/libs/dependencies.bash @@ -0,0 +1,12 @@ +install_yq() { + apk add yq +} + +install_psql() { + apk add postgresql-client +} + +install_dependencies() { + install_yq + install_psql +} diff --git a/scripts/bats/test/libs/partitions.bash b/scripts/bats/test/libs/partitions.bash new file mode 100644 index 0000000..7c191ea --- /dev/null +++ b/scripts/bats/test/libs/partitions.bash @@ -0,0 +1,53 @@ +create_partioned_table() { + local TABLE=$1 + local INTERVAL=$2 + local RETENTION=$3 + local PREPROVISIONED=$4 + + create_table_from_template ${TABLE} + generate_daily_partitions ${TABLE} ${RETENTION} ${PREPROVISIONED} +} + +generate_daily_partitions() { + local PARENT_TABLE=$1 + local RETENTION=$2 + local PREPROVISIONED=$3 + + # Generate retention partitions + for ((i=1; i<=RETENTION; i++)) + do + generate_daily_partition ${PARENT_TABLE} -$i + done + + # Generate current partition + generate_daily_partition ${PARENT_TABLE} 0 + + # Generate preProvisioned partitions + for ((i=1; i<=PREPROVISIONED; i++)) + do + generate_daily_partition ${PARENT_TABLE} $i + done +} + +generate_daily_partition() { + local PARENT_TABLE=$1 + local TIMEDELTA=$2 + + local TABLE_NAME=$(generate_daily_partition_name "${PARENT_TABLE}" "${TIMEDELTA}") + local LOWER_BOUND=$(date -d "@$(( $(date +%s) + 86400 * $TIMEDELTA))" +"%Y-%m-%d") + local UPPER_BOUND=$(date -d "@$(( $(date +%s) + 86400 * $TIMEDELTA + 86400))" +"%Y-%m-%d") + + local QUERY="CREATE TABLE ${TABLE_NAME} PARTITION OF ${PARENT_TABLE} FOR VALUES FROM ('${LOWER_BOUND}') TO ('${UPPER_BOUND}');" + execute_sql "${QUERY}" +} + +generate_daily_partition_name() { + local PARENT_TABLE=$1 + local TIMEDELTA=$2 + + echo $(date -d "@$(( $(date +%s) + 86400 * $TIMEDELTA))" +"${PARENT_TABLE}_%Y_%m_%d") +} + +generate_table_name() { + cat /dev/urandom | head -n 1 | base64 | tr -dc '[:alnum:]' | tr '[:upper:]' '[:lower:]' | cut -c -13 | sed -e 's/^[0-9]/a/g' +} diff --git a/scripts/bats/test/libs/seeds.bash b/scripts/bats/test/libs/seeds.bash new file mode 100644 index 0000000..9776a37 --- /dev/null +++ b/scripts/bats/test/libs/seeds.bash @@ -0,0 +1,40 @@ +init_database() { + QUERY="CREATE DATABASE unittest;" + execute_sql "${QUERY}" postgres +} + +drop_database() { + QUERY="DROP DATABASE IF EXISTS unittest WITH ( FORCE );" + execute_sql "${QUERY}" postgres +} + +reset_database() { + drop_database + init_database +} + +create_table_from_template() { + local TABLE=$1 + + read -r -d '' QUERY < "${TEMPORARY_FILE}" + + local FILENAME=$(mktemp).yaml + yq '. as $item ireduce ({}; . * $item )' "${CONFIGURATION_TEMPLATE_FILE}" "${TEMPORARY_FILE}" > "${FILENAME}" + + echo $FILENAME +} diff --git a/scripts/bats/test/libs/sql.bash b/scripts/bats/test/libs/sql.bash new file mode 100644 index 0000000..758488a --- /dev/null +++ b/scripts/bats/test/libs/sql.bash @@ -0,0 +1,55 @@ +execute_sql() { + local SQL_COMMAND=$1 + local DATABASE=$2 + + if [ -z ${DATABASE} ]; then + psql --echo-all --echo-errors -c "${SQL_COMMAND}" + else + psql --echo-all --echo-errors --dbname="${DATABASE}" -c "${SQL_COMMAND}" + fi +} + +execute_sql_file() { + local SQL_FILE=$1 + local DATABASE=$2 + + if [ -z ${DATABASE} ]; then + psql --echo-all --echo-errors -f "${SQL_FILE}" + else + psql --echo-all --echo-errors --dbname="${DATABASE}" -f "${SQL_FILE}" + fi +} + +assert_table_exists() { + local SCHEMA=$1 + local TABLE=$2 + + local QUERY="SELECT EXISTS ( + SELECT + FROM information_schema.tables + WHERE table_schema = '${SCHEMA}' + AND table_name = '${TABLE}' + );" + + run psql --tuples-only --no-align -c "${QUERY}" + + assert_success + assert_output t +} + +assert_table_not_exists() { + local SCHEMA=$1 + local TABLE=$2 + + local QUERY="SELECT EXISTS ( + SELECT + FROM information_schema.tables + WHERE table_schema = '${SCHEMA}' + AND table_name = '${TABLE}' + );" + + run psql --tuples-only --no-align -c "${QUERY}" + + assert_success + assert_output f +} diff --git a/scripts/localdev/configuration/bats/Dockerfile b/scripts/localdev/configuration/bats/Dockerfile index e27800c..ee73110 100644 --- a/scripts/localdev/configuration/bats/Dockerfile +++ b/scripts/localdev/configuration/bats/Dockerfile @@ -1,3 +1,7 @@ FROM bats/bats:1.11.0 RUN apk add postgresql-client yq + +RUN useradd -m app + +USER app From eca07c7c741b5322bf0931053677093583ee885a Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:19:57 +0200 Subject: [PATCH 13/27] chore(ci): Add test --- .github/workflows/test.yaml | 109 ++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..14e1574 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,109 @@ +--- +name: unittest +on: # yamllint disable-line rule:truthy + push: + branches: + - "*" + +permissions: + contents: read + +jobs: + go: + name: go + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' + - name: Install dependencies + run: | + go get . + - name: Build + run: make build + - name: Run Go tests + run: make test + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + with: + filename: coverage.xml + badge: true + fail_below_min: true + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: '60 80' + - uses: jwalton/gh-find-current-pr@v1 + id: finder + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + with: + number: ${{ steps.finder.outputs.pr }} + path: code-coverage-results.md + recreate: true + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: binary + path: postgresql-partition-manager + if-no-files-found: error + retention-days: 1 + + e2e: + name: End-to-end + needs: go + strategy: + matrix: + postgres: + - postgres:14 + - postgres:15 + - postgres:16 + runs-on: ubuntu-latest + env: + BATS_LIB_PATH: "${{ github.workspace }}/test/bats/lib/" + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: hackme + PGDATABASE: unittest + services: + postgres: + image: ${{ matrix.postgres }} + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --hostname postgres + env: + POSTGRES_PASSWORD: hackme + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Bats and bats libs + uses: bats-core/bats-action@2.0.0 + with: + support-path: ${{ github.workspace }}/test/bats/lib/bats-support + assert-path: ${{ github.workspace }}/test/bats/lib/bats-assert + file-install: false # Unused + detik-install: false # Unused + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: binary + + # File permissions are not maintained during artifact upload/download + - name: Move binary to local executable + run: mv postgresql-partition-manager /usr/local/bin && chmod +x /usr/local/bin/postgresql-partition-manager + + - name: Run bats + run: make bats-test From b70d2901d014e4c5de6ba52fd2e37672dabd5cf0 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:20:05 +0200 Subject: [PATCH 14/27] chore(ci): Add linter --- .github/workflows/linter.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/linter.yaml diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml new file mode 100644 index 0000000..1cc87bc --- /dev/null +++ b/.github/workflows/linter.yaml @@ -0,0 +1,33 @@ +--- +name: linter +on: # yamllint disable-line rule:truthy + push: + branches: + - "*" + +permissions: + contents: read + +jobs: + golangci: + name: golangci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: v1.54 + + yamllint: + name: yamllint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Lint YAML files + run: yamllint . # YAML lint is already installed in ubuntu-latest From abd414763492803c7f7ab0194cd1764a747478a4 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:19:20 +0200 Subject: [PATCH 15/27] chore(docs): Add PPM configuration file template --- .../postgresql-partition-manager.yaml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 configs/postgresql-partition-manager/postgresql-partition-manager.yaml diff --git a/configs/postgresql-partition-manager/postgresql-partition-manager.yaml b/configs/postgresql-partition-manager/postgresql-partition-manager.yaml new file mode 100644 index 0000000..881f25b --- /dev/null +++ b/configs/postgresql-partition-manager/postgresql-partition-manager.yaml @@ -0,0 +1,33 @@ +# +# PostgreSQL partition manager +# + +# Enable debug mode +# debug: false + +# Log format (text or json) +# log-format: json + +# Set maximum allowed duration of any statement (milliseconds unit) +# statement-timeout: 3000 + +# Sets the maximum allowed duration of any wait for a lock (milliseconds unit) +# This parameter prevent infitite locks when long running transaction occured +# lock-timeout: 300 + +# PostgreSQL connection URL +# connection-url: postgres://postgres:root@localhost/partitions + +# +# Partitions definition +# + +# partitions: +# my_partition: +# schema: public +# table: my_date +# partitionKey: my_column +# interval: daily +# retention: 30 +# preProvisioned: 7 +# cleanupPolicy: drop From 1e006316933c68b512728031db82029d294ec06c Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Tue, 16 Apr 2024 17:11:46 +0200 Subject: [PATCH 16/27] chore(README): Initial import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Eve Fritz Co-authored-by: Damien Cupif Co-authored-by: Daniel Vérité --- README.md | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..314fad0 --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +# PostgreSQL Partition Manager + +**PostgreSQL Partition Manager**, or PPM, is an *opinionated tool* designed to streamline the management of PostgreSQL partitions. + +PPM operates on [PostgreSQL's Declarative partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html#DDL-PARTITIONING-DECLARATIVE) and does not require any specific PostgreSQL extensions. + +> [!TIP] +> The key objective of PPM is to simplify the use of PostgreSQL partitions for developers. By providing a secure, non-blocking, and intuitive tool for partition management +> +> Developers can manage table schema and indexes using existing tools and ORMs, and leverate on PPM to implement `date` and [`UUIDv7`](https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis#name-uuid-version-7) column based partitioning for scaling PostgreSQL systems. + +PPM will process all referenced partitions and exit with a non-zero code if it detects an anomaly in at least one partition. This process must be closely monitored to trigger manual intervention, if necessary, to avoid system downtime. + +**Features**: + +- Creation of upcoming partitions +- Delete (or detach) outdated partitions +- Check partitions configuration + +**Opinionated limitations**: + +- Support of PostgreSQL 14+ +- Only supports [`RANGE` partition strategy](https://www.postgresql.org/docs/current/ddl-partitioning.html#DDL-PARTITIONING-OVERVIEW-RANGE) +- The partition key must be a column of `date`, `timestamp`, or `uuid` type +- Support `daily`, `weekly`, `monthly`, and `yearly` partitioning +- Dates are implemented through UTC timezone +- Partition names are enforced and not configurable + + | Partition interval | Pattern | Example | + | ------------------ | --------------------------------- | ----------------- | + | daily | `__
_` | `logs_2024_06_25` | + | weekly | `_w` | `logs_2024_w26` | + | monthly | `__` | `logs_2024_06` | + | yearly | `_` | `logs_2024` | + +## Installation + +PPM is available as a Docker image, Debian package, and Binary. + +
+Docker image + +1. Generate configuration file in `postgresql-partition-manager.yaml` from the docker image + + ```bash + docker run public.ecr.aws/qonto/postgresql-partition-manager:latest -- cat postgresql-partition-manager.yaml + ``` + +1. Launch PPM with a configuration file + + ```bash + docker run -v ./postgresql-partition-manager.yaml:/app/postgresql-partition-manager.yaml public.ecr.aws/qonto/postgresql-partition-manager:latest + ``` + +
+ +
+Debian/Ubuntu + +1. Download the Debian package + + ```bash + POSTGRESQL_PARTITION_MANAGER_VERSION=0.1.0 # Replace with latest version + + PACKAGE_NAME=postgresql_partition_manager_${POSTGRESQL_PARTITION_MANAGER_VERSION}_$(uname -m).deb + wget https://github.com/qonto/postgresql-partition-manager/releases/download/${PROMETHEUS_RDS_EXPORTER_VERSION}/${PACKAGE_NAME} + ``` + +1. Install package + + ```bash + dpkg -i ${PACKAGE_NAME} + ``` + +1. Customize configuration + + Copy configuration file template + + ```bash + cp /usr/share/postgresql-partition-manager/postgresql-partition-manager.yaml.sample postgresql-partition-manager.yaml + ``` + + Edit database connection parameter and partition configuration + + ```bash + vim postgresql-partition-manager.yaml + ``` + +
+ +
+Go + +1. PPM could be installed from Go install + + ```bash + go install github.com/qonto/postgresql-partition-manager@latest + ``` + +
+ +## Usage + +The primary commands include: + +- `validate`: Validates the partition configuration file +- `run check`: Checks if partitions match the expected configuration. This is useful for detecting incorrect partitions +- `run provisioning`: Creates future partitions +- `run cleanup`: Removes (or detach) outdated partitions +- `run all`: Execute provisioning, cleanup, and check commands sequentially + +For a complete list of available commands, use `postgresql-partition-manager --help`. + +> [!TIP] +> We recommend to execute `postgresql-partition-manager run all` every day (e.g., CRON job) with a minimum of 3 pre provisioned partitions (7 for daily partitioning). And raise alerts on non-zero exit code. + +## Configuration + +Configuration could be defined in `postgresql-partition-manager.yaml` or environment variables (format `POSTGRESQL_PARTITION_MANAGER_`). + +| Parameter | Description | Default | +| ----------------- | ---------------------------------------------------- | ------- | +| connection-url | PostgreSQL connection URL | | +| debug | Enable debug mode | false | +| log-format | Log format (text or json) | json | +| lock-timeout | Maximum allowed duration of any wait for a lock (ms) | 300 | +| statement-timeout | Maximum allowed duration of any statement (ms) | 3000 | +| partitions | Map of `` | | + +Partition object: + +| Parameter | Description | Default | +| -------------- | ---------------------------------------------------- | ------- | +| column | Column used for partitioning | | +| interval | Partitioning interval (`daily`, `weekly`, `monthly` or `yearly`) | | +| preProvisioned | Number of partitions to create in advance | | +| retention | Number of partitions to retain | | +| schema | PostgreSQL schema | | +| table | Table to be partitioned | | +| cleanupPolicy | `detach` refers to detaching only the partition while `drop` refers to both detaching and dropping it | | + +See the [full configuration file](configs/postgresql-partition-manager/postgresql-partition-manager.yaml). + +## Contributing + +Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. + +See [Contributing](CONTRIBUTING.md) + +## License + +MIT From 06cedb0cb2afabcd269bd0f29b278f5d1659cbe8 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 07:22:09 +0200 Subject: [PATCH 17/27] chore(helm): Add Helm chart --- .github/workflows/test.yaml | 30 +++- Makefile | 8 + README.md | 111 +++++++++++++- configs/helm/.helmignore | 25 +++ configs/helm/Chart.yaml | 20 +++ configs/helm/LICENSE | 1 + configs/helm/templates/NOTES.txt | 13 ++ configs/helm/templates/_helpers.tpl | 56 +++++++ configs/helm/templates/configmap.yaml | 7 + configs/helm/templates/cronjob.yaml | 108 +++++++++++++ configs/helm/tests/configmap_test.yaml | 19 +++ configs/helm/tests/cronjob_test.yaml | 142 ++++++++++++++++++ .../tests/values/with_additional_labels.yaml | 4 + .../values/with_credentials_in_secret.yaml | 8 + .../values/with_partition_configuration.yaml | 4 + .../helm/tests/values/with_pod_settings.yaml | 28 ++++ configs/helm/tests/values/with_suspend.yaml | 3 + configs/helm/values.yaml | 84 +++++++++++ scripts/kubeconform-test.sh | 60 ++++++++ 19 files changed, 728 insertions(+), 3 deletions(-) create mode 100644 configs/helm/.helmignore create mode 100644 configs/helm/Chart.yaml create mode 120000 configs/helm/LICENSE create mode 100644 configs/helm/templates/NOTES.txt create mode 100644 configs/helm/templates/_helpers.tpl create mode 100644 configs/helm/templates/configmap.yaml create mode 100644 configs/helm/templates/cronjob.yaml create mode 100644 configs/helm/tests/configmap_test.yaml create mode 100644 configs/helm/tests/cronjob_test.yaml create mode 100644 configs/helm/tests/values/with_additional_labels.yaml create mode 100644 configs/helm/tests/values/with_credentials_in_secret.yaml create mode 100644 configs/helm/tests/values/with_partition_configuration.yaml create mode 100644 configs/helm/tests/values/with_pod_settings.yaml create mode 100644 configs/helm/tests/values/with_suspend.yaml create mode 100644 configs/helm/values.yaml create mode 100755 scripts/kubeconform-test.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 14e1574..833cddc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -93,8 +93,8 @@ jobs: with: support-path: ${{ github.workspace }}/test/bats/lib/bats-support assert-path: ${{ github.workspace }}/test/bats/lib/bats-assert - file-install: false # Unused - detik-install: false # Unused + file-install: false # Unused + detik-install: false # Unused - name: Download artifact uses: actions/download-artifact@v4 @@ -107,3 +107,29 @@ jobs: - name: Run bats run: make bats-test + + helm: + name: helm + runs-on: ubuntu-latest + env: + HELM_UNITTEST_VERSION: v0.3.5 + steps: + - uses: actions/checkout@v4 + - name: Install helm-unittest + run: helm plugin install --version $HELM_UNITTEST_VERSION https://github.com/helm-unittest/helm-unittest.git + - name: Run Helm test + run: make helm-test + + kubeconform: + name: kubeconform + runs-on: ubuntu-latest + env: + KUBECONFORM_VERSION: 0.6.2 + steps: + - uses: actions/checkout@v4 + - name: Install kubeconform + run: | + curl -sSLo /tmp/kubeconform.tar.gz "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" \ + && tar -C /usr/local/bin/ -xzvf /tmp/kubeconform.tar.gz + - name: Run Kubeconform test + run: make kubeconform-test diff --git a/Makefile b/Makefile index 2ad23fa..b391471 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,14 @@ install: build bats-test: cd scripts/bats && bats *.bats +.PHONY: helm-test +helm-test: + helm unittest configs/helm + +.PHONY: kubeconform-test +kubeconform-test: + ./scripts/kubeconform-test.sh configs/helm + .PHONY: test test: go test -race -v ./... -coverprofile=coverage.txt -covermode atomic diff --git a/README.md b/README.md index 314fad0..7e8988f 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,115 @@ PPM will process all referenced partitions and exit with a non-zero code if it d PPM is available as a Docker image, Debian package, and Binary. +
+Helm + +1. Create a [Kubernetes secret](https://kubernetes.io/docs/concepts/configuration/secret) containing PostgreSQL password + + While database credentials could be passed in `connection-url` configuration parameter or `PGUSER` and `PGPASSWORD` environment variables. + + We recommend storing credentials in Kubernetes secret and referring to the secret via the `cronjob.postgresqlPasswordSecret` Helm chart parameter. + + Example of creating secret using `kubectl`: + + ```bash + kubectl create secret generic postgresql-credentials --from-literal=password=replace_with_your_postgresql_password + ``` + + We recommend the [Kubernetes Secrets Store CSI driver](https://secrets-store-csi-driver.sigs.k8s.io) for production deployment. + + The `cronjob.postgresqlUserSecret` parameter could be used to pass PostgreSQL user, but don't recommend to store username as a secret because it makes audits more difficult and significantly increases security. + +1. Create a configuration file + + Copy the following template: + + ```bash + cat > values.yaml + cronjob: + postgresqlPasswordSecret: + ref: postgresql-credentials # Specify the Kubernetes secret name containing the PostgreSQL credentials + key: password # Specify the key containing the password + + configuration: + debug: false + + connection-url: postgres://my_username@postgres/my_app # TODO replace with your database connection parameters + + partitions: + #my_partition: + # schema: public + # table: logs + # partitionKey: created_at + # interval: daily + # retention: 30 + # preProvisioned: 7 + # cleanupPolicy: drop + EOF + ``` + + Edit partitioning settings in `partitions`: + + ```bash + vim values.yaml + ``` + +1. Deploy the chart + + Set PPM version to deploy and Kubernetes target namespace: + + ```bash + POSTGRESQL_PARTION_MANAGER=0.1.0 # Replace with latest version + KUBERNETES_NAMESPACE=default # Replace with your namespace + HELM_RELEASE_NAME=main # Replace with an helm release + ``` + + Then deploy it: + + ```bash + helm upgrade \ + ${HELM_RELEASE_NAME} \ + oci://public.ecr.aws/qonto/postgresql-partition-manager-chart \ + --version ${POSTGRESQL_PARTION_MANAGER} \ + --install \ + --namespace ${KUBERNETES_NAMESPACE} + --values + ``` + +1. Trigger job manually and verify application logs + + Set a Kubernetes job name: + + ```bash + MANUAL_JOB=ppm-manually-triggered + ``` + + Trigger job manually: + + ```bash + kubectl create job --namespace ${KUBERNETES_NAMESPACE} --from=cronjob/${HELM_RELEASE_NAME}-postgresql-partition-manager ${MANUAL_JOB} + ``` + + Check cronjob execution: + + ```bash + kubectl describe job --namespace ${KUBERNETES_NAMESPACE} ${MANUAL_JOB} + ``` + + Check application logs + + ```bash + kubectl logs --namespace ${KUBERNETES_NAMESPACE} --selector=job-name=${MANUAL_JOB} + ``` + + Clean up manual job + + ```bash + kubectl delete job --namespace ${KUBERNETES_NAMESPACE} ${MANUAL_JOB} + ``` + +
+
Docker image @@ -63,7 +172,7 @@ PPM is available as a Docker image, Debian package, and Binary. POSTGRESQL_PARTITION_MANAGER_VERSION=0.1.0 # Replace with latest version PACKAGE_NAME=postgresql_partition_manager_${POSTGRESQL_PARTITION_MANAGER_VERSION}_$(uname -m).deb - wget https://github.com/qonto/postgresql-partition-manager/releases/download/${PROMETHEUS_RDS_EXPORTER_VERSION}/${PACKAGE_NAME} + wget https://github.com/qonto/postgresql-partition-manager/releases/download/${POSTGRESQL_PARTION_MANAGER}/${PACKAGE_NAME} ``` 1. Install package diff --git a/configs/helm/.helmignore b/configs/helm/.helmignore new file mode 100644 index 0000000..c2d1ece --- /dev/null +++ b/configs/helm/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# Custom +tests diff --git a/configs/helm/Chart.yaml b/configs/helm/Chart.yaml new file mode 100644 index 0000000..7827234 --- /dev/null +++ b/configs/helm/Chart.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v2 +type: application + +# version and appVersion values are overrided during build step +version: 0.0.0 +appVersion: 0.0.0 + +name: postgresql-partition-manager +description: Manages PostgreSQL partition +home: https://github.com/qonto/postgresql-partition-manager +sources: + - https://github.com/qonto/postgresql-partition-manager +annotations: + artifacthub.io/category: database + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/qonto/postgresql-partition-manager/tree/main/configs/helm +keywords: + - postgresql diff --git a/configs/helm/LICENSE b/configs/helm/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/configs/helm/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/configs/helm/templates/NOTES.txt b/configs/helm/templates/NOTES.txt new file mode 100644 index 0000000..43cfa9f --- /dev/null +++ b/configs/helm/templates/NOTES.txt @@ -0,0 +1,13 @@ +PostgreSQL Partition Manager is deployed. + +1. Get Cronjob + + kubectl --namespace {{ .Release.Namespace }} describe cronjob {{ .Release.Name }}-postgresql-partition-manager + +1. Optional. Manually trigger CRON job manually + + kubectl --namespace {{ .Release.Namespace }} create job --from=cronjob/{{ .Release.Name }}-postgresql-partition-manager ppm-manually-triggered + + And see job execution logs: + + kubectl logs --namespace ${KUBERNETES_NAMESPACE} --selector=job-name=ppm-manually-triggered diff --git a/configs/helm/templates/_helpers.tpl b/configs/helm/templates/_helpers.tpl new file mode 100644 index 0000000..5b73fc6 --- /dev/null +++ b/configs/helm/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "postgresql-partition-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "postgresql-partition-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "postgresql-partition-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "postgresql-partition-manager.labels" -}} +helm.sh/chart: {{ include "postgresql-partition-manager.chart" . }} +{{ include "postgresql-partition-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/component: cronjob +app.kubernetes.io/part-of: postgresql-partition-manager +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- range $key, $value := .Values.additionalLabels }} +{{ $key }}: {{ $value | quote }} +{{- end -}} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "postgresql-partition-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "postgresql-partition-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/configs/helm/templates/configmap.yaml b/configs/helm/templates/configmap.yaml new file mode 100644 index 0000000..5989a49 --- /dev/null +++ b/configs/helm/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-partition-manager.fullname" . }} +data: + configuration: | +{{ .Values.configuration | toYaml | indent 4 }} diff --git a/configs/helm/templates/cronjob.yaml b/configs/helm/templates/cronjob.yaml new file mode 100644 index 0000000..7721adc --- /dev/null +++ b/configs/helm/templates/cronjob.yaml @@ -0,0 +1,108 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "postgresql-partition-manager.fullname" . }} + labels: + {{- include "postgresql-partition-manager.labels" . | nindent 4 }} +spec: + suspend: {{ .Values.cronjob.suspend }} + schedule: {{ .Values.cronjob.schedule | quote }} + timeZone: {{ .Values.cronjob.timeZone }} + successfulJobsHistoryLimit: {{ .Values.cronjob.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.cronjob.failedJobsHistoryLimit }} + concurrencyPolicy: {{ .Values.cronjob.concurrencyPolicy }} + {{- if .Values.cronjob.startingDeadlineSeconds }} + startingDeadlineSeconds: {{ .Values.cronjob.startingDeadlineSeconds }} + {{- end }} + jobTemplate: + spec: + ttlSecondsAfterFinished: {{ .Values.cronjob.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.cronjob.backoffLimit }} + template: + metadata: + labels: + {{- include "postgresql-partition-manager.labels" . | nindent 12 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if not .Values.podAnnotations }} + annotations: + {{- range $key, $val := .Values.podAnnotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + {{- end }} + spec: + automountServiceAccountToken: {{ .Values.cronjob.automountServiceAccountToken }} + terminationGracePeriodSeconds: {{ .Values.cronjob.terminationGracePeriodSeconds }} + {{- if .Values.cronjob.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.cronjob.activeDeadlineSeconds }} + {{- end }} + restartPolicy: {{ .Values.cronjob.restartPolicy }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + containers: + - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.imagePullPolicy }} + name: {{ include "postgresql-partition-manager.name" . }} + env: + {{- range $key, $val := .Values.cronjob.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- if .Values.cronjob.postgresqlUserSecret }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Values.cronjob.postgresqlUserSecret.ref }} + key: {{ .Values.cronjob.postgresqlUserSecret.key }} + {{- end }} + {{- if .Values.cronjob.postgresqlPasswordSecret }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.cronjob.postgresqlPasswordSecret.ref }} + key: {{ .Values.cronjob.postgresqlPasswordSecret.key }} + {{- end }} + {{- if .Values.cronjob.command }} + command: + {{- range .Values.cronjob.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.cronjob.args }} + args: + {{- range .Values.cronjob.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 14 }} + {{- with .Values.cronjob.resources }} + resources: +{{ toYaml . | indent 14 }} + {{- end }} + volumeMounts: + - name: configuration + mountPath: /app/postgresql-partition-manager.yaml + subPath: postgresql-partition-manager.yaml + readOnly: true + volumes: + - name: configuration + configMap: + name: {{ include "postgresql-partition-manager.fullname" . }} + items: + - key: configuration + path: postgresql-partition-manager.yaml + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/configs/helm/tests/configmap_test.yaml b/configs/helm/tests/configmap_test.yaml new file mode 100644 index 0000000..98f38cb --- /dev/null +++ b/configs/helm/tests/configmap_test.yaml @@ -0,0 +1,19 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: configmap tests +templates: + - configmap.yaml +tests: + - it: render default deployment + asserts: + - isKind: + of: ConfigMap + - it: render with partition configuration + values: + - ./values/with_partition_configuration.yaml + asserts: + - equal: + path: data.configuration + value: | + connection-url: postgres://postgres/development + debug: true diff --git a/configs/helm/tests/cronjob_test.yaml b/configs/helm/tests/cronjob_test.yaml new file mode 100644 index 0000000..18a908a --- /dev/null +++ b/configs/helm/tests/cronjob_test.yaml @@ -0,0 +1,142 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: cronjob tests +templates: + - cronjob.yaml +tests: + - it: render default deployment + asserts: + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-postgresql-partition-manager + - equal: + path: spec.suspend + value: false + - equal: + path: spec.schedule + value: "10 0 * * *" + - equal: + path: spec.timeZone + value: "Etc/UTC" + - equal: + path: spec.successfulJobsHistoryLimit + value: 1 + - equal: + path: spec.failedJobsHistoryLimit + value: 1 + - equal: + path: spec.concurrencyPolicy + value: Forbid + - equal: + path: spec.startingDeadlineSeconds + value: 21600 + - equal: + path: spec.jobTemplate.spec.template.spec.restartPolicy + value: Never + - equal: + path: spec.jobTemplate.spec.backoffLimit + value: 0 + - equal: + path: spec.jobTemplate.spec.ttlSecondsAfterFinished + value: 14400 + - equal: + path: spec.jobTemplate.spec.template.spec.terminationGracePeriodSeconds + value: 60 + - notExists: + path: spec.jobTemplate.spec.template.spec.activeDeadlineSeconds + - equal: + path: spec.jobTemplate.spec.template.spec.automountServiceAccountToken + value: false + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].image + value: public.ecr.aws/qonto/postgresql-partition-manager:0.0.0 + - notExists: + path: spec.jobTemplate.spec.template.spec.containers[0].command + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].args + value: ["run", "all"] + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.requests.cpu + value: 10m + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.requests.memory + value: 50Mi + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: 50Mi + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].securityContext.seccompProfile.type + value: RuntimeDefault + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].securityContext.runAsUser + value: 10001 + - it: render disabled cronjob + values: + - ./values/with_suspend.yaml + asserts: + - equal: + path: spec.suspend + value: true + - it: render with credentials in secret + values: + - ./values/with_credentials_in_secret.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[0].name + value: PGUSER + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.name + value: secret-containing-user + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.key + value: user + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[1].name + value: PGPASSWORD + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef.name + value: secret-containing-password + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef.key + value: password + + - it: render disabled cronjob + values: + - ./values/with_pod_settings.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[0].name + value: POSTGRESQL_PARTITION_MANAGER_DEBUG + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].env[0].value + value: "true" + - equal: + path: spec.jobTemplate.spec.template.spec.securityContext.runAsUser + value: 1000 + - equal: + path: spec.jobTemplate.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] + value: linux + - equal: + path: spec.jobTemplate.spec.template.spec.nodeSelector.disktype + value: ssd + - equal: + path: spec.jobTemplate.spec.template.spec.tolerations[0].effect + value: NoSchedule + - it: render with additional labels + values: + - ./values/with_additional_labels.yaml + asserts: + - equal: + path: metadata.labels.label1 + value: value1 + - equal: + path: metadata.labels.label2 + value: value2 diff --git a/configs/helm/tests/values/with_additional_labels.yaml b/configs/helm/tests/values/with_additional_labels.yaml new file mode 100644 index 0000000..0fa9559 --- /dev/null +++ b/configs/helm/tests/values/with_additional_labels.yaml @@ -0,0 +1,4 @@ +--- +additionalLabels: + label1: value1 + label2: value2 diff --git a/configs/helm/tests/values/with_credentials_in_secret.yaml b/configs/helm/tests/values/with_credentials_in_secret.yaml new file mode 100644 index 0000000..96b4ad9 --- /dev/null +++ b/configs/helm/tests/values/with_credentials_in_secret.yaml @@ -0,0 +1,8 @@ +--- +cronjob: + postgresqlUserSecret: + ref: secret-containing-user + key: user + postgresqlPasswordSecret: + ref: secret-containing-password + key: password diff --git a/configs/helm/tests/values/with_partition_configuration.yaml b/configs/helm/tests/values/with_partition_configuration.yaml new file mode 100644 index 0000000..7c40e82 --- /dev/null +++ b/configs/helm/tests/values/with_partition_configuration.yaml @@ -0,0 +1,4 @@ +--- +configuration: + debug: true + connection-url: postgres://postgres/development diff --git a/configs/helm/tests/values/with_pod_settings.yaml b/configs/helm/tests/values/with_pod_settings.yaml new file mode 100644 index 0000000..c746dfb --- /dev/null +++ b/configs/helm/tests/values/with_pod_settings.yaml @@ -0,0 +1,28 @@ +--- +cronjob: + env: + POSTGRESQL_PARTITION_MANAGER_DEBUG: true + +podSecurityContext: + runAsUser: 1000 + +securityContext: + allowPrivilegeEscalation: false + +nodeSelector: + disktype: ssd + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + +tolerations: + - key: "key1" + operator: "Exists" + effect: "NoSchedule" diff --git a/configs/helm/tests/values/with_suspend.yaml b/configs/helm/tests/values/with_suspend.yaml new file mode 100644 index 0000000..86eeb1f --- /dev/null +++ b/configs/helm/tests/values/with_suspend.yaml @@ -0,0 +1,3 @@ +--- +cronjob: + suspend: true diff --git a/configs/helm/values.yaml b/configs/helm/values.yaml new file mode 100644 index 0000000..4fc8a2a --- /dev/null +++ b/configs/helm/values.yaml @@ -0,0 +1,84 @@ +--- +# Default values for postgresql-partition-manager + +image: + repository: public.ecr.aws/qonto/postgresql-partition-manager + pullPolicy: IfNotPresent + tag: "" # Defined by chart appVersion parameter + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} +# fsGroup: 2000 + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 10001 # CKV_K8S_40 Prevent user escalation + seccompProfile: + type: RuntimeDefault +# capabilities: +# drop: +# - ALL +# runAsNonRoot: true +# runAsUser: 1000 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Set additional labels on all resources +additionalLabels: {} + +cronjob: + suspend: false + timeZone: "Etc/UTC" + schedule: "10 0 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + startingDeadlineSeconds: 21600 # This means if the job misses its scheduled time, the system has up to 6 hours to attempt to start the job before it counts as a missed start + restartPolicy: Never + backoffLimit: 0 # 0 means no retry + activeDeadlineSeconds: # Time limit after which the pod will be terminated (SIGTERM) + ttlSecondsAfterFinished: 14400 # 4 hours + terminationGracePeriodSeconds: 60 + automountServiceAccountToken: false + command: [] + args: + - run + - all + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + memory: 50Mi + +configuration: +# partitions: +# by_date: +# schema: public +# table: by_date +# partitionKey: created_at +# interval: yearly +# retention: 7 +# preProvisioned: 7 +# cleanupPolicy: drop diff --git a/scripts/kubeconform-test.sh b/scripts/kubeconform-test.sh new file mode 100755 index 0000000..012366e --- /dev/null +++ b/scripts/kubeconform-test.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# This script tests that all Helm chart test values and default chart values return a valid Kubernetes manifest + +set -e +set -o pipefail + +HELM_CHART_DIRECTORY=$1 +KUBERNETES_VERSION=${KUBERNETES_VERSION-1.25.0} +KUBECONFORM_CACHE_DIRECTORY=${KUBECONFORM_CACHE_DIRECTORY-/tmp} + +HELM_DEFAULT_VALUE_FILE=${HELM_CHART_DIRECTORY}/values.yaml +HELM_TEST_VALUES_DIRECTORY=${HELM_CHART_DIRECTORY}/tests/values + +check_parameters() { + if [[ -z $HELM_CHART_DIRECTORY ]]; then + echo "ERRROR: You must specify the helm chart directory" + usage + fi + if [[ ! -f $HELM_DEFAULT_VALUE_FILE ]]; then + echo "ERRROR: Default Helm values ${HELM_DEFAULT_VALUE_FILE} does not exists" + usage + fi + if [[ ! -d $HELM_TEST_VALUES_DIRECTORY ]]; then + echo "ERRROR: Helm test values directory ${HELM_TEST_VALUES_DIRECTORY} does not exists" + usage + fi +} + +usage() { + echo "" + echo "Usage: $0 " + exit 1 +} + +check_parameters + +HELM_VALUE_FILES=$(find ${HELM_DEFAULT_VALUE_FILE} ${HELM_TEST_VALUES_DIRECTORY}/*.yaml) + +for FILE in $HELM_VALUE_FILES; +do + printf "\033[32mTest chart with ${FILE}\033[0m\n" + echo "" + + # Redirect Helm error output to /dev/null to remove symlink warning, see https://github.com/helm/helm/issues/7019 + helm template configs/helm \ + -f $FILE \ + 2> /dev/null \ + | kubeconform \ + --strict \ + -exit-on-error \ + -kubernetes-version ${KUBERNETES_VERSION} \ + -cache ${KUBECONFORM_CACHE_DIRECTORY} \ + -schema-location default \ + -schema-location 'scripts/kubeconform/{{.Group}}/{{ .ResourceKind }}_{{.ResourceAPIVersion}}.json' \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + -summary + + echo "" +done From ddf3d8dacb55a21fab7b335bdebb709c74a6b7c0 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 13:50:42 +0200 Subject: [PATCH 18/27] chore(debian): Add Debian package --- .github/workflows/test.yaml | 22 +++++++++++++++ Makefile | 15 +++++++++- configs/debian/tests/Dockerfile | 19 +++++++++++++ configs/debian/tests/sudoers | 2 ++ configs/debian/tests/test.bats | 49 +++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 configs/debian/tests/Dockerfile create mode 100644 configs/debian/tests/sudoers create mode 100755 configs/debian/tests/test.bats diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 833cddc..2ffe32b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -133,3 +133,25 @@ jobs: && tar -C /usr/local/bin/ -xzvf /tmp/kubeconform.tar.gz - name: Run Kubeconform test run: make kubeconform-test + + debian: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version: stable + - name: Set up QEMU for ARM64 build + uses: docker/setup-qemu-action@v3 + - uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean --skip=publish --skip=docker --snapshot + env: + GORELEASER_CURRENT_TAG: 0.0.0 + - name: Run Debian package tests + run: make debian-test-ci diff --git a/Makefile b/Makefile index b391471..c58548f 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,19 @@ helm-test: kubeconform-test: ./scripts/kubeconform-test.sh configs/helm +.PHONY: goreleaser-check +goreleaser-check: + goreleaser check + +debian-test: + GORELEASER_CURRENT_TAG=0.0.0 goreleaser release --clean --skip-publish --skip-docker --snapshot + docker build configs/debian/tests -t test + docker run -v ./dist/postgresql-partition-manager_0.0.1~next_$(ARCHITECTURE).deb:/mnt/postgresql-partition-manager.deb test + +debian-test-ci: + docker build configs/debian/tests -t test + docker run -v ./dist/postgresql-partition-manager_0.0.1~next_amd64.deb:/mnt/postgresql-partition-manager.deb test + .PHONY: test test: go test -race -v ./... -coverprofile=coverage.txt -covermode atomic @@ -46,4 +59,4 @@ lint: golangci-lint run --verbose --timeout 2m .PHONY: all-tests -all-tests: test goreleaser-check +all-tests: test helm-test kubeconform-test goreleaser-check diff --git a/configs/debian/tests/Dockerfile b/configs/debian/tests/Dockerfile new file mode 100644 index 0000000..6446aa8 --- /dev/null +++ b/configs/debian/tests/Dockerfile @@ -0,0 +1,19 @@ +#checkov:skip=CKV2_DOCKER_1:Sudo is required to test installation and suppression of the exporter Debian package + +FROM debian:bookworm + +HEALTHCHECK NONE + +RUN apt-get update \ + && apt-get install -y bats bats-assert bats-file sudo \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY sudoers /etc/sudoers.d/ + +RUN useradd -m unittest --groups sudo + +USER unittest + +COPY . /tmp/ + +CMD [ "/tmp/test.bats" ] diff --git a/configs/debian/tests/sudoers b/configs/debian/tests/sudoers new file mode 100644 index 0000000..b638b6b --- /dev/null +++ b/configs/debian/tests/sudoers @@ -0,0 +1,2 @@ +# Allow unittest user to install/remove package +unittest ALL = (ALL:ALL) NOPASSWD: /usr/bin/dpkg -i /mnt/postgresql-partition-manager.deb, /usr/bin/apt-get remove -y postgresql-partition-manager, /usr/bin/apt-get purge -y postgresql-partition-manager diff --git a/configs/debian/tests/test.bats b/configs/debian/tests/test.bats new file mode 100755 index 0000000..72b37dc --- /dev/null +++ b/configs/debian/tests/test.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats + +load '/usr/lib/bats/bats-support/load' +load '/usr/lib/bats/bats-assert/load' +load '/usr/lib/bats/bats-file/load' + +PACKAGE=/mnt/postgresql-partition-manager.deb + +setup() { + run bash -c "DEBIAN_FRONTEND=noninteractive sudo dpkg -i ${PACKAGE}" + assert_success +} + +remove_package() { + run bash -c 'sudo apt-get remove -y postgresql-partition-manager' + assert_success +} + +purge_package() { + run bash -c 'sudo apt-get purge -y postgresql-partition-manager' + assert_success +} + +@test "Test installation" { + assert_file_exist /usr/share/postgresql-partition-manager/postgresql-partition-manager.yaml.sample + + run bash -c 'postgresql-partition-manager --version' + assert_success + assert_output --regexp '^postgresql-partition-manager version' + + run bash -c "dpkg --info ${PACKAGE}" + assert_output --regexp 'Package: postgresql-partition-manager' + assert_output --regexp 'Streamline the management of PostgreSQL partitions' + assert_output --regexp 'Maintainer: SRE Team' + assert_output --regexp 'Homepage: https://github.com/qonto/postgresql-partition-manager' +} + +@test 'Check removed package' { + remove_package + + assert_file_not_exist /usr/bin/postgresql-partition-manager +} + +@test 'Check purged package' { + purge_package + + assert_file_not_exist /usr/bin/postgresql-partition-manager + assert_file_not_exist /usr/share/postgresql-partition-manager/postgresql-partition-manager.yaml.sample +} From 3fd5df6973aca7c2b0d7a4e0cb204c7b6eec907c Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 15:20:49 +0200 Subject: [PATCH 19/27] chore(go): Simplify date in postgresql tests --- .../postgresql/partition_internal_test.go | 30 ++-- internal/infra/postgresql/partition_test.go | 132 ++++++++++-------- internal/infra/uuid7/uuid7_test.go | 22 +-- 3 files changed, 101 insertions(+), 83 deletions(-) diff --git a/internal/infra/postgresql/partition_internal_test.go b/internal/infra/postgresql/partition_internal_test.go index 0649f18..ba8c8cf 100644 --- a/internal/infra/postgresql/partition_internal_test.go +++ b/internal/infra/postgresql/partition_internal_test.go @@ -11,8 +11,8 @@ func TestParseBounds(t *testing.T) { testCases := []struct { name string partition Partition - lowerbound time.Time - upperBound time.Time + lowerbound string + upperBound string }{ { "Date bounds", @@ -22,8 +22,8 @@ func TestParseBounds(t *testing.T) { LowerBound: "2024-01-01", UpperBound: "2025-03-02", }, - time.Date(2024, 1, 1, 0, 0, 0, 0, time.Now().UTC().Location()), - time.Date(2025, 3, 2, 0, 0, 0, 0, time.Now().UTC().Location()), + "2024-01-01T00:00:00Z", + "2025-03-02T00:00:00Z", }, { "Datetime bounds", @@ -33,8 +33,8 @@ func TestParseBounds(t *testing.T) { LowerBound: "2024-01-01 10:00:00", UpperBound: "2025-02-03 12:53:00", }, - time.Date(2024, 1, 1, 10, 0, 0, 0, time.Now().UTC().Location()), - time.Date(2025, 2, 3, 12, 53, 0, 0, time.Now().UTC().Location()), + "2024-01-01T10:00:00Z", + "2025-02-03T12:53:00Z", }, { "UUIDv7 bounds", @@ -44,8 +44,8 @@ func TestParseBounds(t *testing.T) { LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 }, - time.Date(2024, 1, 1, 0, 0, 0, 0, time.Now().UTC().Location()), - time.Date(2024, 1, 2, 0, 0, 0, 0, time.Now().UTC().Location()), + "2024-01-01T00:00:00Z", + "2024-01-02T00:00:00Z", }, { "Native Time.time bounds", @@ -55,18 +55,24 @@ func TestParseBounds(t *testing.T) { LowerBound: time.Date(2024, 1, 1, 10, 12, 5, 0, time.Now().UTC().Location()), UpperBound: time.Date(2025, 2, 3, 12, 53, 35, 0, time.Now().UTC().Location()), }, - time.Date(2024, 1, 1, 10, 12, 5, 0, time.Now().UTC().Location()), - time.Date(2025, 2, 3, 12, 53, 35, 0, time.Now().UTC().Location()), + "2024-01-01T10:12:05Z", + "2025-02-03T12:53:35Z", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + expectedLowerbound, err := time.Parse(time.RFC3339, tc.lowerbound) + assert.NilError(t, err, "LowerBound parsing failed") + + expectedUpperBound, err := time.Parse(time.RFC3339, tc.upperBound) + assert.NilError(t, err, "Upperbound parsing failed") + lowerBound, upperBound, err := parseBounds(tc.partition) assert.NilError(t, err, "Bounds parsing should succeed") - assert.Equal(t, lowerBound, tc.lowerbound, "LowerBound mismatch") - assert.Equal(t, upperBound, tc.upperBound, "UpperBound mismatch") + assert.Equal(t, lowerBound, expectedLowerbound, "LowerBound mismatch") + assert.Equal(t, upperBound, expectedUpperBound, "UpperBound mismatch") }) } } diff --git a/internal/infra/postgresql/partition_test.go b/internal/infra/postgresql/partition_test.go index 5f48ad6..643134a 100644 --- a/internal/infra/postgresql/partition_test.go +++ b/internal/infra/postgresql/partition_test.go @@ -66,12 +66,12 @@ func TestPartitionName(t *testing.T) { testCases := []struct { name string partition postgresql.PartitionConfiguration - time time.Time + when string expected postgresql.Partition }{ { - name: "Daily partition", - partition: postgresql.PartitionConfiguration{ + "Daily partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -80,8 +80,8 @@ func TestPartitionName(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), - expected: postgresql.Partition{ + "2024-01-30T12:53:45Z", + postgresql.Partition{ Schema: "public", Name: "my_table_2024_01_30", LowerBound: time.Date(2024, 0o1, 30, 0, 0, 0, 0, time.UTC), @@ -89,8 +89,8 @@ func TestPartitionName(t *testing.T) { }, }, { - name: "Monthly partition", - partition: postgresql.PartitionConfiguration{ + "Monthly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -99,8 +99,8 @@ func TestPartitionName(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), - expected: postgresql.Partition{ + "2024-01-30T12:53:45Z", + postgresql.Partition{ Schema: "public", Name: "my_table_2024_01", LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), @@ -108,8 +108,8 @@ func TestPartitionName(t *testing.T) { }, }, { - name: "Weekly partition", - partition: postgresql.PartitionConfiguration{ + "Weekly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -118,8 +118,8 @@ func TestPartitionName(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), - expected: postgresql.Partition{ + "2024-01-30T12:53:45Z", + postgresql.Partition{ Schema: "public", Name: "my_table_2024_w05", LowerBound: time.Date(2024, 0o1, 29, 0, 0, 0, 0, time.UTC), @@ -127,8 +127,8 @@ func TestPartitionName(t *testing.T) { }, }, { - name: "Yearly partition", - partition: postgresql.PartitionConfiguration{ + "Yearly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -137,8 +137,8 @@ func TestPartitionName(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 30, 12, 53, 45, 100, time.UTC), - expected: postgresql.Partition{ + "2024-01-30T12:53:45Z", + postgresql.Partition{ Schema: "public", Name: "my_table_2024", LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), @@ -149,7 +149,11 @@ func TestPartitionName(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result, _ := tc.partition.GeneratePartition(tc.time) + when, err := time.Parse(time.RFC3339, tc.when) + assert.NilError(t, err, "Time parse failed") + + result, err := tc.partition.GeneratePartition(when) + assert.NilError(t, err, "Generate partition failed") assert.Equal(t, tc.expected.Schema, result.Schema, "Schema don't match") assert.Equal(t, tc.expected.Name, result.Name, "Table name don't match") assert.Equal(t, tc.expected.LowerBound, result.LowerBound, "Lower bound don't match") @@ -162,12 +166,12 @@ func TestRetentionTableNames(t *testing.T) { testCases := []struct { name string partition postgresql.PartitionConfiguration - time time.Time + when string expected []postgresql.Partition }{ { - name: "Daily partition", - partition: postgresql.PartitionConfiguration{ + "Daily partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -176,8 +180,8 @@ func TestRetentionTableNames(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 0o3, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2024-01-03T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2024_01_02", @@ -206,8 +210,8 @@ func TestRetentionTableNames(t *testing.T) { }, }, { - name: "Monthly partition", - partition: postgresql.PartitionConfiguration{ + "Monthly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -216,8 +220,8 @@ func TestRetentionTableNames(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o2, 25, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2024-02-25T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2024_01", @@ -240,8 +244,8 @@ func TestRetentionTableNames(t *testing.T) { }, }, { - name: "Weekly partition", - partition: postgresql.PartitionConfiguration{ + "Weekly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -250,8 +254,8 @@ func TestRetentionTableNames(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 9, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2024-01-09T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2024_w01", @@ -268,8 +272,8 @@ func TestRetentionTableNames(t *testing.T) { }, }, { - name: "Yearly partition", - partition: postgresql.PartitionConfiguration{ + "Yearly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -278,8 +282,8 @@ func TestRetentionTableNames(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 9, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2024-01-09T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2023", @@ -296,8 +300,8 @@ func TestRetentionTableNames(t *testing.T) { }, }, { - name: "No retention", - partition: postgresql.PartitionConfiguration{ + "No retention", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -306,15 +310,17 @@ func TestRetentionTableNames(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 9, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{}, + "2024-01-09T12:53:45Z", + []postgresql.Partition{}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tables, _ := tc.partition.GetRetentionPartitions(tc.time) + when, err := time.Parse(time.RFC3339, tc.when) + assert.NilError(t, err, "Time parse failed") + tables, _ := tc.partition.GetRetentionPartitions(when) assert.DeepEqual(t, tables, tc.expected) }) } @@ -324,12 +330,12 @@ func TestPreProvisionedTableNames(t *testing.T) { testCases := []struct { name string partition postgresql.PartitionConfiguration - time time.Time + when string expected []postgresql.Partition }{ { - name: "Daily partition", - partition: postgresql.PartitionConfiguration{ + "Daily partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -338,8 +344,8 @@ func TestPreProvisionedTableNames(t *testing.T) { PreProvisioned: 4, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2024, 0o1, 29, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2024-01-29T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2024_01_30", @@ -368,8 +374,8 @@ func TestPreProvisionedTableNames(t *testing.T) { }, }, { - name: "Monthly partition", - partition: postgresql.PartitionConfiguration{ + "Monthly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -378,8 +384,8 @@ func TestPreProvisionedTableNames(t *testing.T) { PreProvisioned: 3, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2023, 11, 29, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2023-11-29T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2023_12", @@ -402,8 +408,8 @@ func TestPreProvisionedTableNames(t *testing.T) { }, }, { - name: "Weekly partition", - partition: postgresql.PartitionConfiguration{ + "Weekly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -412,8 +418,8 @@ func TestPreProvisionedTableNames(t *testing.T) { PreProvisioned: 2, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2023, 12, 20, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2023-12-20T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2023_w52", @@ -430,8 +436,8 @@ func TestPreProvisionedTableNames(t *testing.T) { }, }, { - name: "Yearly partition", - partition: postgresql.PartitionConfiguration{ + "Yearly partition", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -440,8 +446,8 @@ func TestPreProvisionedTableNames(t *testing.T) { PreProvisioned: 2, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2023, 12, 20, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{ + "2023-12-10T12:53:45Z", + []postgresql.Partition{ { Schema: "public", Name: "my_table_2024", @@ -458,8 +464,8 @@ func TestPreProvisionedTableNames(t *testing.T) { }, }, { - name: "No PreProvisioned", - partition: postgresql.PartitionConfiguration{ + "No PreProvisioned", + postgresql.PartitionConfiguration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -468,15 +474,17 @@ func TestPreProvisionedTableNames(t *testing.T) { PreProvisioned: 0, CleanupPolicy: postgresql.DropCleanupPolicy, }, - time: time.Date(2023, 12, 20, 12, 53, 45, 100, time.UTC), - expected: []postgresql.Partition{}, + "2023-12-20T12:53:45Z", + []postgresql.Partition{}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tables, _ := tc.partition.GetPreProvisionedPartitions(tc.time) + when, err := time.Parse(time.RFC3339, tc.when) + assert.NilError(t, err, "Time parse failed") + tables, _ := tc.partition.GetPreProvisionedPartitions(when) assert.DeepEqual(t, tables, tc.expected) }) } diff --git a/internal/infra/uuid7/uuid7_test.go b/internal/infra/uuid7/uuid7_test.go index ecf036f..45d3bc7 100644 --- a/internal/infra/uuid7/uuid7_test.go +++ b/internal/infra/uuid7/uuid7_test.go @@ -13,27 +13,31 @@ const UUIDv7Version uuid.Version = 7 func TestFromTime(t *testing.T) { testCases := []struct { - timestamp time.Time + timestamp string expected string }{ - { - time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC), - "018d242a-c800-7000-0000-000000000000", - }, + {"2023-12-31T23:59:59Z", "018cc251-f018-7000-0000-000000000000"}, + {"2024-01-01T00:00:00Z", "018cc251-f400-7000-0000-000000000000"}, + {"2024-01-02T12:45:35Z", "018cca35-3998-7000-0000-000000000000"}, + {"2020-02-29T01:00:00Z", "01708e75-0a80-7000-0000-000000000000"}, // leap year + {"1996-12-19T16:39:57-08:00", "00c62614-6b48-7000-0000-000000000000"}, // 20 minutes and 50.52 seconds after the 23rd hour of April 12th, 1985 in UTC. } for _, tc := range testCases { - t.Run(tc.timestamp.String(), func(t *testing.T) { - generated := uuid7.FromTime(tc.timestamp) + t.Run(tc.timestamp, func(t *testing.T) { + timestamp, err := time.Parse(time.RFC3339, tc.timestamp) + assert.Nil(t, err, "Time parse failed") + + generated := uuid7.FromTime(timestamp) assert.Equal(t, generated, tc.expected, "Should match expected ") decoded, err := uuid.Parse(generated) - timestamp, _ := decoded.Time().UnixTime() + decodedTimestamp, _ := decoded.Time().UnixTime() assert.Nil(t, err, "UUID should be parsable") assert.Equal(t, decoded.Version(), UUIDv7Version, "Should be an UUIDv7") - assert.Equal(t, timestamp, tc.timestamp.Unix(), "Timestamp from generated UUID should match") + assert.Equal(t, timestamp.Unix(), decodedTimestamp, "Timestamp from generated UUID should match") }) } } From 8ef42b24a360cda21816fbfd497ab474208c6d59 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Mon, 22 Apr 2024 18:19:09 +0200 Subject: [PATCH 20/27] refact(postgresql): Rewrite package to split PostgreSQL logic --- internal/infra/postgresql/bounds.go | 141 ----- internal/infra/postgresql/column.go | 35 +- .../infra/postgresql/column_internal_test.go | 30 +- internal/infra/postgresql/column_test.go | 32 - internal/infra/postgresql/connection.go | 1 - internal/infra/postgresql/error.go | 17 - .../infra/postgresql/error_internal_test.go | 45 -- internal/infra/postgresql/partition.go | 143 ++++- .../postgresql/partition_internal_test.go | 150 ----- internal/infra/postgresql/partition_test.go | 597 ++++-------------- .../postgresql/partitionconfiguration.go | 150 ----- .../infra/postgresql/partitionsettings.go | 23 - .../postgresql/partitionsettings_test.go | 63 -- .../infra/postgresql/partitionstrategy.go | 9 - internal/infra/postgresql/postgres.go | 32 + internal/infra/postgresql/postgresql.go | 366 ----------- .../postgresql/postgresql_internal_test.go | 272 -------- internal/infra/postgresql/postgresql_test.go | 35 + internal/infra/postgresql/server.go | 17 +- internal/infra/postgresql/server_test.go | 61 +- internal/infra/postgresql/table.go | 43 +- internal/infra/postgresql/table_test.go | 82 ++- pkg/ppm/server_test.go | 2 +- 23 files changed, 498 insertions(+), 1848 deletions(-) delete mode 100644 internal/infra/postgresql/bounds.go delete mode 100644 internal/infra/postgresql/column_test.go delete mode 100644 internal/infra/postgresql/error.go delete mode 100644 internal/infra/postgresql/error_internal_test.go delete mode 100644 internal/infra/postgresql/partition_internal_test.go delete mode 100644 internal/infra/postgresql/partitionconfiguration.go delete mode 100644 internal/infra/postgresql/partitionsettings.go delete mode 100644 internal/infra/postgresql/partitionsettings_test.go delete mode 100644 internal/infra/postgresql/partitionstrategy.go create mode 100644 internal/infra/postgresql/postgres.go delete mode 100644 internal/infra/postgresql/postgresql.go delete mode 100644 internal/infra/postgresql/postgresql_internal_test.go create mode 100644 internal/infra/postgresql/postgresql_test.go diff --git a/internal/infra/postgresql/bounds.go b/internal/infra/postgresql/bounds.go deleted file mode 100644 index 859abfa..0000000 --- a/internal/infra/postgresql/bounds.go +++ /dev/null @@ -1,141 +0,0 @@ -package postgresql - -import ( - "errors" - "fmt" - "time" - - "github.com/google/uuid" -) - -const ( - UUIDv7Version uuid.Version = 7 -) - -var ( - ErrLowerBoundAfterUpperBound = errors.New("lowerbound is after upperbound") - - // ErrCantDecodePartitionBounds represents an error indicating that the partition bounds cannot be decoded. - ErrCantDecodePartitionBounds = errors.New("partition bounds cannot be decoded") - - ErrUnsupportedUUIDVersion = errors.New("unsupported UUID version") -) - -func getDailyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, date.UTC().Location()) - upperBound = lowerBound.AddDate(0, 0, 1) - - return -} - -func getWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, date.UTC().Location()).AddDate(0, 0, -int(date.Weekday()-time.Monday)) - upperBound = lowerBound.AddDate(0, 0, daysInAweek) - - return -} - -func getMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, date.UTC().Location()) - upperBound = lowerBound.AddDate(0, 1, 0) - - return -} - -func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { - lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, date.UTC().Location()) - upperBound = lowerBound.AddDate(1, 0, 0) - - return -} - -func parseBounds(partition Partition) (lowerBound time.Time, upperBound time.Time, err error) { - lowerBound, upperBound, err = parseBoundAsTime(partition) - if err == nil { - return lowerBound, upperBound, nil - } - - lowerBound, upperBound, err = parseBoundAsDate(partition) - if err == nil { - return lowerBound, upperBound, nil - } - - lowerBound, upperBound, err = parseBoundAsDateTime(partition) - if err == nil { - return lowerBound, upperBound, nil - } - - lowerBound, upperBound, err = parseBoundAsUUIDv7(partition) - if err == nil { - return lowerBound, upperBound, nil - } - - if lowerBound.After(lowerBound) { - return time.Time{}, time.Time{}, ErrLowerBoundAfterUpperBound - } - - return time.Time{}, time.Time{}, ErrCantDecodePartitionBounds -} - -func parseBoundAsTime(partition Partition) (lowerBound, upperBound time.Time, err error) { - lowerBound, ok := partition.LowerBound.(time.Time) - if !ok { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as time: %w", err) - } - - upperBound, ok = partition.UpperBound.(time.Time) - if !ok { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as time: %w", err) - } - - return lowerBound, upperBound, nil -} - -func parseBoundAsDate(partition Partition) (lowerBound, upperBound time.Time, err error) { - lowerBound, err = time.Parse("2006-01-02", partition.LowerBound.(string)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as date: %w", err) - } - - upperBound, err = time.Parse("2006-01-02", partition.UpperBound.(string)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as date: %w", err) - } - - return lowerBound, upperBound, nil -} - -func parseBoundAsDateTime(partition Partition) (lowerBound, upperBound time.Time, err error) { - lowerBound, err = time.Parse("2006-01-02 15:04:05", partition.LowerBound.(string)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime: %w", err) - } - - upperBound, err = time.Parse("2006-01-02 15:04:05", partition.UpperBound.(string)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime: %w", err) - } - - return lowerBound, upperBound, nil -} - -func parseBoundAsUUIDv7(partition Partition) (lowerBound, upperBound time.Time, err error) { - lowerBoundUUID, err := uuid.Parse(partition.LowerBound.(string)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as UUID: %w", err) - } - - upperBoundUUID, err := uuid.Parse(partition.UpperBound.(string)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as UUID: %w", err) - } - - if upperBoundUUID.Version() != UUIDv7Version || lowerBoundUUID.Version() != UUIDv7Version { - return time.Time{}, time.Time{}, ErrUnsupportedUUIDVersion - } - - upperBound = time.Unix(upperBoundUUID.Time().UnixTime()).UTC() - lowerBound = time.Unix(lowerBoundUUID.Time().UnixTime()).UTC() - - return lowerBound, upperBound, nil -} diff --git a/internal/infra/postgresql/column.go b/internal/infra/postgresql/column.go index 5aed69a..eb382fa 100644 --- a/internal/infra/postgresql/column.go +++ b/internal/infra/postgresql/column.go @@ -3,13 +3,6 @@ package postgresql import ( "errors" "fmt" - "strings" -) - -const ( - DateColumnType ColumnType = "date" - DateTimeColumnType ColumnType = "timestamp" - UUIDColumnType ColumnType = "uuid" ) // ErrUnsupportedPartitionKeyType represents an error indicating that the column type for partitioning is not supported. @@ -17,19 +10,13 @@ var ErrUnsupportedPartitionKeyType = errors.New("unsupported partition key colum type ColumnType string -type Column struct { - Schema string - Table string - Name string - DataType ColumnType -} - -func (c Column) String() string { - return strings.Join([]string{c.Schema, c.Table, c.Name}, ".") -} +const ( + Date ColumnType = "date" + DateTime ColumnType = "timestamp" + UUID ColumnType = "uuid" +) -// Return the PostgreSQL data type of the specified column -func (p PostgreSQL) getColumnDataType(column Column) (ColumnType, error) { +func (p Postgres) GetColumnDataType(schema, table, column string) (ColumnType, error) { var columnType string query := `SELECT @@ -40,20 +27,20 @@ func (p PostgreSQL) getColumnDataType(column Column) (ColumnType, error) { AND table_name = $2 AND column_name = $3` - err := p.db.QueryRow(p.ctx, query, column.Schema, column.Table, column.Name).Scan(&columnType) + err := p.conn.QueryRow(p.ctx, query, schema, table, column).Scan(&columnType) if err != nil { return "", fmt.Errorf("failed to get %s column type: %w", column, err) } switch columnType { case "date": - return DateColumnType, nil + return Date, nil case "timestamp": - return DateTimeColumnType, nil + return DateTime, nil case "timestamp without time zone": - return DateTimeColumnType, nil + return DateTime, nil case "uuid": - return UUIDColumnType, nil + return UUID, nil default: return "", fmt.Errorf("%w: %s", ErrUnsupportedPartitionKeyType, columnType) } diff --git a/internal/infra/postgresql/column_internal_test.go b/internal/infra/postgresql/column_internal_test.go index 73dc97a..a1e0359 100644 --- a/internal/infra/postgresql/column_internal_test.go +++ b/internal/infra/postgresql/column_internal_test.go @@ -1,7 +1,6 @@ package postgresql import ( - "fmt" "testing" "github.com/pashagolub/pgxmock/v3" @@ -10,24 +9,20 @@ import ( ) func TestGetColumn(t *testing.T) { - column := Column{ - Schema: "public", - Table: "my_table", - Name: "my_column", - } + schema := "public" + table := "my_table" + column := "my_column" const query = `SELECT data_type as columnType FROM information_schema.columns WHERE table_schema = \$1 AND table_name = \$2 AND column_name = \$3` mock, err := pgxmock.NewConn() if err != nil { - fmt.Println("ERROR: Fail to initialize PostgreSQL mock: %w", err) - panic(err) + t.Fatalf("ERROR: Fail to initialize PostgreSQL mock: %s", err) } logger, err := logger.New(false, "text") if err != nil { - fmt.Println("ERROR: Fail to initialize logger: %w", err) - panic(err) + t.Fatalf("ERROR: Fail to initialize logger: %s", err) } p := New(*logger, mock) @@ -40,24 +35,29 @@ func TestGetColumn(t *testing.T) { { "Date", "date", - DateColumnType, + Date, }, { "Date time", + "timestamp", + DateTime, + }, + { + "Date time without time zone", "timestamp without time zone", - DateTimeColumnType, + DateTime, }, { "UUID", "uuid", - UUIDColumnType, + UUID, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - mock.ExpectQuery(query).WithArgs(column.Schema, column.Table, column.Name).WillReturnRows(mock.NewRows([]string{"columnType"}).AddRow(tc.postgreSQLcolumn)) - dataType, err := p.getColumnDataType(column) + mock.ExpectQuery(query).WithArgs(schema, table, column).WillReturnRows(mock.NewRows([]string{"columnType"}).AddRow(tc.postgreSQLcolumn)) + dataType, err := p.GetColumnDataType(schema, table, column) assert.Nil(t, err, "getColumnDataType should succeed") assert.Equal(t, dataType, tc.dataType, "Column type should match") diff --git a/internal/infra/postgresql/column_test.go b/internal/infra/postgresql/column_test.go deleted file mode 100644 index 18b18f9..0000000 --- a/internal/infra/postgresql/column_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package postgresql_test - -import ( - "testing" - - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" - "gotest.tools/assert" -) - -func TestColumnAttributes(t *testing.T) { - testCases := []struct { - name string - column postgresql.Column - expectedName string - }{ - { - name: "Public schema", - column: postgresql.Column{ - Schema: "public", - Table: "my_table", - Name: "my_column", - }, - expectedName: "public.my_table.my_column", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.column.String(), tc.expectedName, "Column name don't match") - }) - } -} diff --git a/internal/infra/postgresql/connection.go b/internal/infra/postgresql/connection.go index 90d9d1c..a6b2b92 100644 --- a/internal/infra/postgresql/connection.go +++ b/internal/infra/postgresql/connection.go @@ -1,4 +1,3 @@ -// Package postgresql provides methods to interact with PostgreSQL internal resources (tables, columns, ...) package postgresql import ( diff --git a/internal/infra/postgresql/error.go b/internal/infra/postgresql/error.go deleted file mode 100644 index ec74bdf..0000000 --- a/internal/infra/postgresql/error.go +++ /dev/null @@ -1,17 +0,0 @@ -package postgresql - -import ( - "errors" - - "github.com/jackc/pgx/v5/pgconn" -) - -const ( - ObjectNotInPrerequisiteStatePostgreSQLErrorCode = "55000" -) - -func isPostgreSQLErrorCode(err error, errorCode string) bool { - var pgErr *pgconn.PgError - - return errors.As(err, &pgErr) && pgErr.Code == errorCode -} diff --git a/internal/infra/postgresql/error_internal_test.go b/internal/infra/postgresql/error_internal_test.go deleted file mode 100644 index a55ef48..0000000 --- a/internal/infra/postgresql/error_internal_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package postgresql - -import ( - "errors" - "testing" - - "github.com/jackc/pgx/v5/pgconn" - "gotest.tools/assert" -) - -var ErrGeneric = errors.New("a generic error") - -func TestPostgreSQLError(t *testing.T) { - testCases := []struct { - name string - error error - code string - expected bool - }{ - { - name: "ObjectNotInPrerequisiteState", - error: &pgconn.PgError{Code: "55000"}, - code: "55000", - expected: true, - }, - { - name: "Non match error code", - error: &pgconn.PgError{Code: "42"}, - code: "55000", - expected: false, - }, - { - name: "Generic error", - error: ErrGeneric, - code: "", - expected: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, isPostgreSQLErrorCode(tc.error, tc.code), tc.expected, "Error code should match") - }) - } -} diff --git a/internal/infra/postgresql/partition.go b/internal/infra/postgresql/partition.go index f681221..5a51d42 100644 --- a/internal/infra/postgresql/partition.go +++ b/internal/infra/postgresql/partition.go @@ -1,35 +1,142 @@ package postgresql -import "fmt" +import ( + "errors" + "fmt" -type Partition struct { + "github.com/jackc/pgx/v5" +) + +var ( + // ErrUnsupportedPartitionStrategy represents an error indicating that the partitioning strategy on the table is not supported. + ErrUnsupportedPartitionStrategy = errors.New("unsupported partitioning strategy") + + // ErrTableIsNotPartioned represents an error indicating the specified table don't have partitioning + ErrTableIsNotPartioned = errors.New("table is not partioned") +) + +type PartitionResult struct { ParentTable string Schema string Name string - LowerBound interface{} - UpperBound interface{} + LowerBound string + UpperBound string +} + +func (p Postgres) IsPartitionAttached(schema, table string) (exists bool, err error) { + query := `SELECT EXISTS( + SELECT 1 FROM pg_inherits WHERE inhrelid = $1::regclass + )` + + err = p.conn.QueryRow(p.ctx, query, fmt.Sprintf("%s.%s", schema, table)).Scan(&exists) + if err != nil { + return false, fmt.Errorf("failed to check partition attachment: %w", err) + } + + return exists, nil } -func (p Partition) String() string { - return p.QualifiedName() +func (p Postgres) AttachPartition(schema, table, parent, lowerBound, upperBound string) error { + query := fmt.Sprintf("ALTER TABLE %s.%s ATTACH PARTITION %s.%s FOR VALUES FROM ('%s') TO ('%s')", schema, parent, schema, table, lowerBound, upperBound) + p.logger.Debug("Attach partition", "query", query, "schema", schema, "table", table) + + _, err := p.conn.Exec(p.ctx, query) + if err != nil { + return fmt.Errorf("failed to attach partition: %w", err) + } + + return nil +} + +// DetachPartitionConcurrently detaches specified partition from the parent table. +// The partition still exists as standalone table after detaching +// More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION +func (p Postgres) DetachPartitionConcurrently(schema, table, parent string) error { + query := fmt.Sprintf(`ALTER TABLE %s.%s DETACH PARTITION %s.%s CONCURRENTLY`, schema, parent, schema, table) + p.logger.Debug("Detach partition", "schema", schema, "table", table, "query", query, "parent_table", parent) + + _, err := p.conn.Exec(p.ctx, query) + if err != nil { + return fmt.Errorf("failed to detach partition from the parent table: %w", err) + } + + return nil } -// QualifiedName returns the fully qualified name of the partition (format: .
) -// This is recommended to avoid schema conflicts when querying PostgreSQL catalog tables -func (p Partition) QualifiedName() string { - return fmt.Sprintf("%s.%s", p.Schema, p.Name) +// FinalizePartitionDetach finalizes a partition detach operation. +// It's required when a partition is in "detach pending" status. +// More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION +func (p Postgres) FinalizePartitionDetach(schema, table, parent string) error { + query := fmt.Sprintf(`ALTER TABLE %s.%s DETACH PARTITION %s.%s FINALIZE`, schema, parent, schema, table) + p.logger.Debug("finialize detach partition", "schema", schema, "table", table, "query", query, "parent_table", parent) + + _, err := p.conn.Exec(p.ctx, query) + if err != nil { + return fmt.Errorf("failed to finalize partition detach: %w", err) + } + + return nil } -func (p Partition) ToTable() Table { - return Table{ - Schema: p.Schema, - Name: p.Name, +func (p Postgres) ListPartitions(schema, table string) (partitions []PartitionResult, err error) { + query := fmt.Sprintf(` + WITH parts as ( + SELECT + relnamespace::regnamespace as schema, + c.oid::pg_catalog.regclass AS part_name, + regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FOR VALUES FROM \(''(.*)''\) TO \(''(.*)''\)') AS bounds + FROM + pg_catalog.pg_class c JOIN pg_catalog.pg_inherits i ON (c.oid = i.inhrelid) + WHERE i.inhparent = '%s.%s'::regclass + AND c.relkind='r' + ) + SELECT + schema, + part_name as name, + '%s' as parentTable, + bounds[1]::text AS lowerBound, + bounds[2]::text AS upperBound + FROM parts + ORDER BY part_name;`, schema, table, table) + + rows, err := p.conn.Query(p.ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to list partitions: %w", err) + } + + partitions, err = pgx.CollectRows(rows, pgx.RowToStructByName[PartitionResult]) + if err != nil { + return nil, fmt.Errorf("failed to cast list: %w", err) } + + return partitions, nil } -func (p Partition) GetParentTable() Table { - return Table{ - Schema: p.Schema, - Name: p.ParentTable, +func (p Postgres) GetPartitionSettings(schema, table string) (strategy, key string, err error) { + var partkeydef []string + + // pg_get_partkeydef() is a system function returning the definition of a partitioning key + // It return a text string: () + // Example for RANGE (created_at) + query := fmt.Sprintf(` + SELECT regexp_match(partkeydef, '^(.*) \((.*)\)$') + FROM pg_catalog.pg_get_partkeydef('%s.%s'::regclass) as partkeydef + `, schema, table) + + err = p.conn.QueryRow(p.ctx, query).Scan(&partkeydef) + if err != nil { + p.logger.Warn("failed to get partitioning key", "error", err, "schema", schema, "table", table) + + return "", "", fmt.Errorf("failed to get partition key: %w", err) } + + if len(partkeydef) == 0 { + return "", "", ErrTableIsNotPartioned + } + + strategy = partkeydef[0] + key = partkeydef[1] + + return strategy, key, nil } diff --git a/internal/infra/postgresql/partition_internal_test.go b/internal/infra/postgresql/partition_internal_test.go deleted file mode 100644 index ba8c8cf..0000000 --- a/internal/infra/postgresql/partition_internal_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package postgresql - -import ( - "testing" - "time" - - "gotest.tools/assert" -) - -func TestParseBounds(t *testing.T) { - testCases := []struct { - name string - partition Partition - lowerbound string - upperBound string - }{ - { - "Date bounds", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "2024-01-01", - UpperBound: "2025-03-02", - }, - "2024-01-01T00:00:00Z", - "2025-03-02T00:00:00Z", - }, - { - "Datetime bounds", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "2024-01-01 10:00:00", - UpperBound: "2025-02-03 12:53:00", - }, - "2024-01-01T10:00:00Z", - "2025-02-03T12:53:00Z", - }, - { - "UUIDv7 bounds", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 - UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 - }, - "2024-01-01T00:00:00Z", - "2024-01-02T00:00:00Z", - }, - { - "Native Time.time bounds", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: time.Date(2024, 1, 1, 10, 12, 5, 0, time.Now().UTC().Location()), - UpperBound: time.Date(2025, 2, 3, 12, 53, 35, 0, time.Now().UTC().Location()), - }, - "2024-01-01T10:12:05Z", - "2025-02-03T12:53:35Z", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - expectedLowerbound, err := time.Parse(time.RFC3339, tc.lowerbound) - assert.NilError(t, err, "LowerBound parsing failed") - - expectedUpperBound, err := time.Parse(time.RFC3339, tc.upperBound) - assert.NilError(t, err, "Upperbound parsing failed") - - lowerBound, upperBound, err := parseBounds(tc.partition) - - assert.NilError(t, err, "Bounds parsing should succeed") - assert.Equal(t, lowerBound, expectedLowerbound, "LowerBound mismatch") - assert.Equal(t, upperBound, expectedUpperBound, "UpperBound mismatch") - }) - } -} - -func TestParseInvalidBounds(t *testing.T) { - testCases := []struct { - name string - partition Partition - }{ - { - "UUID v1 upper bound", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 - UpperBound: "47568e76-fb49-11ee-b9c7-325096b39f47", // UUIDv1 - }, - }, - { - "UUID v1 lower bound", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "ad5dac7a-fb46-11ee-be67-325096b39f47", // UUIDv1 - UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 - }, - }, - { - "Mix date format", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "2024-01-01", - UpperBound: "2024-01-02 00:00:00", - }, - }, - { - "Mix date and UUIDv7", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "2024-01-01", - UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 - }, - }, - { - "Mix date and UUIDv7", - Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "2024-01-01", - UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - _, _, err := parseBounds(tc.partition) - - assert.ErrorContains(t, err, "partition bounds cannot be decoded") - }) - } -} - -func TestDebug(t *testing.T) { - partition := Partition{ - Schema: "public", - Name: "my_table", - LowerBound: "2024-01-01 10:00:00", - UpperBound: "2024-01-02 14:00:00", - } - _, _, err := parseBoundAsDateTime(partition) - assert.NilError(t, err) -} diff --git a/internal/infra/postgresql/partition_test.go b/internal/infra/postgresql/partition_test.go index 643134a..ab1ec27 100644 --- a/internal/infra/postgresql/partition_test.go +++ b/internal/infra/postgresql/partition_test.go @@ -1,491 +1,158 @@ +//nolint:golint,wsl,goconst package postgresql_test import ( + "fmt" "testing" - "time" + "github.com/pashagolub/pgxmock/v3" "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" - "gotest.tools/assert" + "github.com/stretchr/testify/assert" ) -func TestPartitionAttributes(t *testing.T) { - testCases := []struct { - name string - partition postgresql.Partition - expectedName string - expectedTable postgresql.Table - expectedParentTable postgresql.Table - }{ - { - name: "Public schema", - partition: postgresql.Partition{ - ParentTable: "my_table", - Schema: "public", - Name: "my_table_2024_12_25", - }, - expectedName: "public.my_table_2024_12_25", - expectedTable: postgresql.Table{ - Schema: "public", - Name: "my_table_2024_12_25", - }, - expectedParentTable: postgresql.Table{ - Schema: "public", - Name: "my_table", - }, - }, - { - name: "Dashed table", - partition: postgresql.Partition{ - ParentTable: "my-table", - Schema: "api", - Name: "my-table_2024_w01", - }, - expectedName: "api.my-table_2024_w01", - expectedTable: postgresql.Table{ - Schema: "api", - Name: "my-table_2024_w01", - }, - expectedParentTable: postgresql.Table{ - Schema: "api", - Name: "my-table", - }, - }, - } +func generateTable(t *testing.T) (schema, table, fullQualifiedTable, parent string) { + t.Helper() - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.partition.QualifiedName(), tc.expectedName, "Qualified name don't match") - assert.Equal(t, tc.partition.String(), tc.expectedName, "Partition name don't match") - assert.Equal(t, tc.partition.ToTable(), tc.expectedTable, "Table don't match") - assert.Equal(t, tc.partition.GetParentTable(), tc.expectedParentTable, "Parent table don't match") - }) - } + schema = "public" + table = "my_table" + fullQualifiedTable = fmt.Sprintf("%s.%s", schema, table) + parent = "my_parent_table" + + return } -func TestPartitionName(t *testing.T) { - testCases := []struct { - name string - partition postgresql.PartitionConfiguration - when string - expected postgresql.Partition - }{ - { - "Daily partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.DailyInterval, - Retention: 7, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-30T12:53:45Z", - postgresql.Partition{ - Schema: "public", - Name: "my_table_2024_01_30", - LowerBound: time.Date(2024, 0o1, 30, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 31, 0, 0, 0, 0, time.UTC), - }, - }, - { - "Monthly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.MonthlyInterval, - Retention: 7, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-30T12:53:45Z", - postgresql.Partition{ - Schema: "public", - Name: "my_table_2024_01", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, - { - "Weekly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.WeeklyInterval, - Retention: 7, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-30T12:53:45Z", - postgresql.Partition{ - Schema: "public", - Name: "my_table_2024_w05", - LowerBound: time.Date(2024, 0o1, 29, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o5, 0, 0, 0, 0, time.UTC), - }, - }, - { - "Yearly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.YearlyInterval, - Retention: 7, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-30T12:53:45Z", - postgresql.Partition{ - Schema: "public", - Name: "my_table_2024", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2025, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, - } +func TestIsPartitionAttached(t *testing.T) { + schema, table, fullQualifiedTable, _ := generateTable(t) - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - when, err := time.Parse(time.RFC3339, tc.when) - assert.NilError(t, err, "Time parse failed") - - result, err := tc.partition.GeneratePartition(when) - assert.NilError(t, err, "Generate partition failed") - assert.Equal(t, tc.expected.Schema, result.Schema, "Schema don't match") - assert.Equal(t, tc.expected.Name, result.Name, "Table name don't match") - assert.Equal(t, tc.expected.LowerBound, result.LowerBound, "Lower bound don't match") - assert.Equal(t, tc.expected.UpperBound, result.UpperBound, "Upper bound don't match") - }) - } + mock, p := setupMock(t, pgxmock.QueryMatcherRegexp) + query := "SELECT EXISTS" + + mock.ExpectQuery(query).WithArgs(fullQualifiedTable).WillReturnRows(mock.NewRows([]string{"EXISTS"}).AddRow(true)) + exists, err := p.IsPartitionAttached(schema, table) + assert.Nil(t, err, "IsPartitionAttached should succeed") + assert.True(t, exists, "Table should be attached") + + mock.ExpectQuery(query).WithArgs(fullQualifiedTable).WillReturnRows(mock.NewRows([]string{"EXISTS"}).AddRow(false)) + exists, err = p.IsPartitionAttached(schema, table) + assert.Nil(t, err, "IsPartitionAttached should succeed") + assert.False(t, exists, "Table should not be attached") + + mock.ExpectQuery(query).WithArgs(fullQualifiedTable).WillReturnError(ErrPostgreSQLConnectionFailure) + _, err = p.IsPartitionAttached(schema, table) + assert.Error(t, err, "IsPartitionAttached should fail") } -func TestRetentionTableNames(t *testing.T) { - testCases := []struct { - name string - partition postgresql.PartitionConfiguration - when string - expected []postgresql.Partition - }{ - { - "Daily partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.DailyInterval, - Retention: 4, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-03T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2024_01_02", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o2, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o3, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_01_01", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o2, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2023_12_31", - ParentTable: "my_table", - LowerBound: time.Date(2023, 12, 31, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2023_12_30", - ParentTable: "my_table", - LowerBound: time.Date(2023, 12, 30, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2023, 12, 31, 0, 0, 0, 0, time.UTC), - }, - }, - }, - { - "Monthly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.MonthlyInterval, - Retention: 3, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-02-25T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2024_01", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2023_12", - ParentTable: "my_table", - LowerBound: time.Date(2023, 12, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2023_11", - ParentTable: "my_table", - LowerBound: time.Date(2023, 11, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2023, 12, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, - }, - { - "Weekly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.WeeklyInterval, - Retention: 2, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-09T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2024_w01", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 8, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2023_w52", - ParentTable: "my_table", - LowerBound: time.Date(2023, 12, 25, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, - }, - { - "Yearly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.YearlyInterval, - Retention: 2, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-09T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2023", - ParentTable: "my_table", - LowerBound: time.Date(2023, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2022", - ParentTable: "my_table", - LowerBound: time.Date(2022, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2023, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, - }, - { - "No retention", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.WeeklyInterval, - Retention: 0, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-09T12:53:45Z", - []postgresql.Partition{}, - }, - } +func TestAttachPartition(t *testing.T) { + schema, table, _, parent := generateTable(t) + lowerBound := "2024-01-30" + upperBound := "2024-01-31" - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - when, err := time.Parse(time.RFC3339, tc.when) - assert.NilError(t, err, "Time parse failed") + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + query := fmt.Sprintf(`ALTER TABLE %s.%s ATTACH PARTITION %s.%s FOR VALUES FROM ('%s') TO ('%s')`, schema, parent, schema, table, lowerBound, upperBound) - tables, _ := tc.partition.GetRetentionPartitions(when) - assert.DeepEqual(t, tables, tc.expected) - }) - } + mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("ALTER", 1)) + err := p.AttachPartition(schema, table, parent, lowerBound, upperBound) + assert.Nil(t, err, "AttachPartition should succeed") + + mock.ExpectExec(query).WillReturnError(ErrPostgreSQLConnectionFailure) + err = p.AttachPartition(schema, table, parent, lowerBound, upperBound) + assert.Error(t, err, "AttachPartition should fail") } -func TestPreProvisionedTableNames(t *testing.T) { - testCases := []struct { - name string - partition postgresql.PartitionConfiguration - when string - expected []postgresql.Partition - }{ - { - "Daily partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.DailyInterval, - Retention: 4, - PreProvisioned: 4, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2024-01-29T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2024_01_30", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 30, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 31, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_01_31", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 31, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_02_01", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o2, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_02_02", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o2, 0o2, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o3, 0, 0, 0, 0, time.UTC), - }, - }, - }, - { - "Monthly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.MonthlyInterval, - Retention: 3, - PreProvisioned: 3, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2023-11-29T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2023_12", - ParentTable: "my_table", - LowerBound: time.Date(2023, 12, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_01", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_02", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o2, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o3, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, - }, - { - "Weekly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.WeeklyInterval, - Retention: 2, - PreProvisioned: 2, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2023-12-20T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2023_w52", - ParentTable: "my_table", - LowerBound: time.Date(2023, 12, 25, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2024_w01", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2024, 0o1, 8, 0, 0, 0, 0, time.UTC), - }, - }, - }, +func TestDetachPartitionConcurrently(t *testing.T) { + schema, table, _, parent := generateTable(t) + + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + query := fmt.Sprintf(`ALTER TABLE %s.%s DETACH PARTITION %s.%s CONCURRENTLY`, schema, parent, schema, table) + + mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("ALTER", 1)) + err := p.DetachPartitionConcurrently(schema, table, parent) + assert.Nil(t, err, "AttachPartition should succeed") + + mock.ExpectExec(query).WillReturnError(ErrPostgreSQLConnectionFailure) + err = p.DetachPartitionConcurrently(schema, table, parent) + assert.Error(t, err, "AttachPartition should fail") +} + +func TestFinalizePartitionDetach(t *testing.T) { + schema, table, _, parent := generateTable(t) + + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + + query := fmt.Sprintf(`ALTER TABLE %s.%s DETACH PARTITION %s.%s FINALIZE`, schema, parent, schema, table) + + mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("ALTER", 1)) + err := p.FinalizePartitionDetach(schema, table, parent) + assert.Nil(t, err, "AttachPartition should succeed") + + mock.ExpectExec(query).WillReturnError(ErrPostgreSQLConnectionFailure) + err = p.FinalizePartitionDetach(schema, table, parent) + assert.Error(t, err, "AttachPartition should fail") +} + +func TestGetPartitionSettings(t *testing.T) { + schema, table, _, _ := generateTable(t) + expectedStrategy := "RANGE" + expectedKey := "created_at" + + mock, p := setupMock(t, pgxmock.QueryMatcherRegexp) + + query := `SELECT regexp_match` + + mock.ExpectQuery(query).WillReturnRows(mock.NewRows([]string{"partkeydef"}).AddRow([]string{expectedStrategy, expectedKey})) + strategy, key, err := p.GetPartitionSettings(schema, table) + assert.Nil(t, err, "GetPartitionSettings should succeed") + assert.Equal(t, strategy, expectedStrategy, "Strategy should match") + assert.Equal(t, key, expectedKey, "Key should match") + + mock.ExpectQuery(query).WillReturnRows(mock.NewRows([]string{"partkeydef"}).AddRow([]string{})) + _, _, err = p.GetPartitionSettings(schema, table) + assert.Error(t, err, "GetPartitionSettings should fail") + assert.ErrorIs(t, err, postgresql.ErrTableIsNotPartioned) + + mock.ExpectQuery(query).WillReturnError(ErrPostgreSQLConnectionFailure) + _, _, err = p.GetPartitionSettings(schema, table) + assert.Error(t, err, "GetPartitionSettings should fail") +} + +func TestListPartitions(t *testing.T) { + schema, table, parent, _ := generateTable(t) + + mock, p := setupMock(t, pgxmock.QueryMatcherRegexp) + query := `WITH parts as` + + expectedPartitions := []postgresql.PartitionResult{ { - "Yearly partition", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.YearlyInterval, - Retention: 2, - PreProvisioned: 2, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2023-12-10T12:53:45Z", - []postgresql.Partition{ - { - Schema: "public", - Name: "my_table_2024", - ParentTable: "my_table", - LowerBound: time.Date(2024, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2025, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, { - Schema: "public", - Name: "my_table_2025", - ParentTable: "my_table", - LowerBound: time.Date(2025, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - UpperBound: time.Date(2026, 0o1, 0o1, 0, 0, 0, 0, time.UTC), - }, - }, + Schema: schema, + ParentTable: parent, + Name: fmt.Sprintf("%s_%s", parent, "2024_01_29"), + LowerBound: "2024-01-29", + UpperBound: "2024-01-30", }, { - "No PreProvisioned", - postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.WeeklyInterval, - Retention: 2, - PreProvisioned: 0, - CleanupPolicy: postgresql.DropCleanupPolicy, - }, - "2023-12-20T12:53:45Z", - []postgresql.Partition{}, + Schema: table, + ParentTable: parent, + Name: fmt.Sprintf("%s_%s", parent, "2024_01_30"), + LowerBound: "2024-01-30", + UpperBound: "2024-01-31", }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - when, err := time.Parse(time.RFC3339, tc.when) - assert.NilError(t, err, "Time parse failed") - - tables, _ := tc.partition.GetPreProvisionedPartitions(when) - assert.DeepEqual(t, tables, tc.expected) - }) + rows := mock.NewRows([]string{"schema", "name", "parentTable", "lowerBound", "upperBound"}) + for _, p := range expectedPartitions { + rows.AddRow(p.Schema, p.Name, p.ParentTable, p.LowerBound, p.UpperBound) } + mock.ExpectQuery(query).WillReturnRows(rows) + result, err := p.ListPartitions(schema, table) + assert.Nil(t, err, "ListPartitions should succeed") + assert.Equal(t, result, expectedPartitions, "Partitions should be match") + + rows = mock.NewRows([]string{"invalidColumn"}).AddRow("invalidColumn") + mock.ExpectQuery(query).WillReturnRows(rows) + _, err = p.ListPartitions(schema, table) + assert.Error(t, err, "ListPartitions should fail") + + mock.ExpectQuery(query).WillReturnError(ErrPostgreSQLConnectionFailure) + _, err = p.ListPartitions(schema, table) + assert.Error(t, err, "ListPartitions should fail") } diff --git a/internal/infra/postgresql/partitionconfiguration.go b/internal/infra/postgresql/partitionconfiguration.go deleted file mode 100644 index ca4f474..0000000 --- a/internal/infra/postgresql/partitionconfiguration.go +++ /dev/null @@ -1,150 +0,0 @@ -package postgresql - -import ( - "errors" - "fmt" - "time" -) - -type ( - Interval string - CleanupPolicy string -) - -const ( - DailyInterval Interval = "daily" - WeeklyInterval Interval = "weekly" - MonthlyInterval Interval = "monthly" - YearlyInterval Interval = "yearly" - DropCleanupPolicy CleanupPolicy = "drop" - DetachCleanupPolicy CleanupPolicy = "detach" - daysInAweek int = 7 -) - -var ErrUnsupportedInterval = errors.New("unsupported partition interval") - -type PartitionConfiguration struct { - Schema string `mapstructure:"schema" validate:"required"` - Table string `mapstructure:"table" validate:"required"` - PartitionKey string `mapstructure:"partitionKey" validate:"required"` - Interval Interval `mapstructure:"interval" validate:"required,oneof=daily weekly monthly yearly"` - Retention int `mapstructure:"retention" validate:"required,gt=0"` - PreProvisioned int `mapstructure:"preProvisioned" validate:"required,gt=0"` - CleanupPolicy CleanupPolicy `mapstructure:"cleanupPolicy" validate:"required,oneof=drop detach"` -} - -func (p PartitionConfiguration) GeneratePartition(forDate time.Time) (Partition, error) { - var suffix string - - var lowerBound, upperBound any - - switch p.Interval { - case DailyInterval: - suffix = forDate.Format("2006_01_02") - lowerBound, upperBound = getDailyBounds(forDate) - case WeeklyInterval: - year, week := forDate.ISOWeek() - suffix = fmt.Sprintf("%d_w%02d", year, week) - lowerBound, upperBound = getWeeklyBounds(forDate) - case MonthlyInterval: - suffix = forDate.Format("2006_01") - lowerBound, upperBound = getMonthlyBounds(forDate) - case YearlyInterval: - suffix = forDate.Format("2006") - lowerBound, upperBound = getYearlyBounds(forDate) - default: - return Partition{}, ErrUnsupportedInterval - } - - partition := Partition{ - Schema: p.Schema, - ParentTable: p.Table, - Name: fmt.Sprintf("%s_%s", p.Table, suffix), - LowerBound: lowerBound, - UpperBound: upperBound, - } - - return partition, nil -} - -func (p PartitionConfiguration) GetRetentionPartitions(forDate time.Time) ([]Partition, error) { - partitions := make([]Partition, p.Retention) - - for i := 1; i <= p.Retention; i++ { - prevDate, err := p.getPrevDate(forDate, i) - if err != nil { - return nil, fmt.Errorf("could not compute previous date: %w", err) - } - - partition, err := p.GeneratePartition(prevDate) - if err != nil { - return nil, fmt.Errorf("could not generate partition: %w", err) - } - - partitions[i-1] = partition - } - - return partitions, nil -} - -func (p PartitionConfiguration) GetPreProvisionedPartitions(forDate time.Time) ([]Partition, error) { - partitions := make([]Partition, p.PreProvisioned) - - for i := 1; i <= p.PreProvisioned; i++ { - nextDate, err := p.getNextDate(forDate, i) - if err != nil { - return nil, fmt.Errorf("could not compute next date: %w", err) - } - - partition, err := p.GeneratePartition(nextDate) - if err != nil { - return nil, fmt.Errorf("could not generate partition: %w", err) - } - - partitions[i-1] = partition - } - - return partitions, nil -} - -func (p PartitionConfiguration) getPrevDate(forDate time.Time, i int) (t time.Time, err error) { - switch p.Interval { - case DailyInterval: - t = forDate.AddDate(0, 0, -i) - case WeeklyInterval: - t = forDate.AddDate(0, 0, -i*daysInAweek) - case MonthlyInterval: - year, month, _ := forDate.Date() - - t = time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, forDate.Location()) - case YearlyInterval: - year, _, _ := forDate.Date() - - t = time.Date(year-i, 1, 1, 0, 0, 0, 0, forDate.Location()) - default: - return time.Time{}, ErrUnsupportedInterval - } - - return t, nil -} - -func (p PartitionConfiguration) getNextDate(forDate time.Time, i int) (t time.Time, err error) { - switch p.Interval { - case DailyInterval: - t = forDate.AddDate(0, 0, i) - case WeeklyInterval: - t = forDate.AddDate(0, 0, i*daysInAweek) - case MonthlyInterval: - year, month, _ := forDate.Date() - - t = time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, forDate.Location()) - case YearlyInterval: - year, _, _ := forDate.Date() - - t = time.Date(year+i, 1, 1, 0, 0, 0, 0, forDate.Location()) - default: - return time.Time{}, ErrUnsupportedInterval - } - - return t, nil -} diff --git a/internal/infra/postgresql/partitionsettings.go b/internal/infra/postgresql/partitionsettings.go deleted file mode 100644 index f7320b9..0000000 --- a/internal/infra/postgresql/partitionsettings.go +++ /dev/null @@ -1,23 +0,0 @@ -package postgresql - -type PartitionSettings struct { - Strategy PartitionStrategy - Key string - KeyType ColumnType -} - -func (p PartitionSettings) SupportedStrategy() bool { - return p.Strategy == RangePartitionStrategy -} - -func (p PartitionSettings) SupportedKeyDataType() bool { - switch p.KeyType { - case - DateColumnType, - DateTimeColumnType, - UUIDColumnType: - return true - } - - return false -} diff --git a/internal/infra/postgresql/partitionsettings_test.go b/internal/infra/postgresql/partitionsettings_test.go deleted file mode 100644 index ce4df05..0000000 --- a/internal/infra/postgresql/partitionsettings_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package postgresql_test - -import ( - "testing" - - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" - "gotest.tools/assert" -) - -func TestPartitionSettings(t *testing.T) { - var UnsupportedColumnType postgresql.ColumnType = "unsupported" - - testCases := []struct { - name string - partition postgresql.PartitionSettings - supportedKeyDataType bool - supportedStrategy bool - }{ - { - name: "Public", - partition: postgresql.PartitionSettings{ - Strategy: postgresql.RangePartitionStrategy, - KeyType: postgresql.DateColumnType, - }, - supportedStrategy: true, - supportedKeyDataType: true, - }, - { - name: "Unsupported list strategy", - partition: postgresql.PartitionSettings{ - Strategy: postgresql.ListPartitionStrategy, - KeyType: postgresql.DateColumnType, - }, - supportedKeyDataType: true, - supportedStrategy: false, - }, - { - name: "Unsupported hash strategy", - partition: postgresql.PartitionSettings{ - Strategy: postgresql.HashPartitionStrategy, - KeyType: postgresql.DateColumnType, - }, - supportedKeyDataType: true, - supportedStrategy: false, - }, - { - name: "Unsupported column type", - partition: postgresql.PartitionSettings{ - Strategy: postgresql.RangePartitionStrategy, - KeyType: UnsupportedColumnType, - }, - supportedKeyDataType: false, - supportedStrategy: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.partition.SupportedStrategy(), tc.supportedStrategy, "Supported strategy mismatch") - assert.Equal(t, tc.partition.SupportedKeyDataType(), tc.supportedKeyDataType, "Supported key data type mismatch") - }) - } -} diff --git a/internal/infra/postgresql/partitionstrategy.go b/internal/infra/postgresql/partitionstrategy.go deleted file mode 100644 index 44adc45..0000000 --- a/internal/infra/postgresql/partitionstrategy.go +++ /dev/null @@ -1,9 +0,0 @@ -package postgresql - -const ( - RangePartitionStrategy PartitionStrategy = "RANGE" - ListPartitionStrategy PartitionStrategy = "LIST" - HashPartitionStrategy PartitionStrategy = "HASH" -) - -type PartitionStrategy string diff --git a/internal/infra/postgresql/postgres.go b/internal/infra/postgresql/postgres.go new file mode 100644 index 0000000..534f6fd --- /dev/null +++ b/internal/infra/postgresql/postgres.go @@ -0,0 +1,32 @@ +// Package postgresql provides methods to interact with PostgreSQL server +package postgresql + +import ( + "context" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type PgxIface interface { + Close(context.Context) error + Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) + PgConn() *pgconn.PgConn + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +type Postgres struct { + ctx context.Context + conn PgxIface + logger slog.Logger +} + +func New(logger slog.Logger, conn PgxIface) *Postgres { + return &Postgres{ + ctx: context.TODO(), + conn: conn, + logger: logger, + } +} diff --git a/internal/infra/postgresql/postgresql.go b/internal/infra/postgresql/postgresql.go deleted file mode 100644 index 058f6b0..0000000 --- a/internal/infra/postgresql/postgresql.go +++ /dev/null @@ -1,366 +0,0 @@ -package postgresql - -import ( - "context" - "errors" - "fmt" - "log/slog" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" - "github.com/qonto/postgresql-partition-manager/internal/infra/retry" - "github.com/qonto/postgresql-partition-manager/internal/infra/uuid7" -) - -// ErrUnsupportedPartitionStrategy represents an error indicating that the partitioning strategy on the table is not supported. -var ( - ErrUnsupportedPartitionStrategy = errors.New("unsupported partitioning strategy") - ErrPartitionNotFound = errors.New("partition not found") -) - -type PostgreSQL struct { - ctx context.Context - db PgxIface - logger slog.Logger -} - -type PgxIface interface { - Close(context.Context) error - Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) - QueryRow(ctx context.Context, sql string, args ...any) pgx.Row - Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) - PgConn() *pgconn.PgConn -} - -func New(logger slog.Logger, db PgxIface) *PostgreSQL { - return &PostgreSQL{ - ctx: context.TODO(), - db: db, - logger: logger, - } -} - -func (p PostgreSQL) CreatePartition(partitionConfiguration PartitionConfiguration, partition Partition) error { - p.logger.Debug("Creating partition", "schema", partition.Schema, "table", partition.Name) - - tableExists, err := p.tableExists(partition.ToTable()) - if err != nil { - return fmt.Errorf("failed to check if table exists: %w", err) - } - - if !tableExists { - query := fmt.Sprintf("CREATE TABLE %s (LIKE %s)", partition.QualifiedName(), partition.ParentTable) - p.logger.Debug("Create table", "query", query) - - _, err := p.db.Exec(context.Background(), query) - if err != nil { - return fmt.Errorf("failed to create table: %w", err) - } - - p.logger.Info("Table created", "schema", partition.Schema, "table", partition.Name) - } else { - p.logger.Info("Table already exists, skip", "schema", partition.Schema, "table", partition.Name) - } - - lowerBoundTime, upperBoundTime, err := parseBounds(partition) - if err != nil { - return fmt.Errorf("failed to bounds: %w", err) - } - - partitionSettings, err := p.GetPartitionSettings(partition.GetParentTable()) - if err != nil { - return fmt.Errorf("failed to get partition settings: %w", err) - } - - switch partitionSettings.KeyType { - case DateColumnType: - partition.LowerBound = lowerBoundTime.Format("2006-01-02") - partition.UpperBound = upperBoundTime.Format("2006-01-02") - case DateTimeColumnType: - partition.LowerBound = lowerBoundTime.Format("2006-01-02 00:00:00") - partition.UpperBound = upperBoundTime.Format("2006-01-02 00:00:00") - case UUIDColumnType: - partition.LowerBound = uuid7.FromTime(lowerBoundTime) - partition.UpperBound = uuid7.FromTime(upperBoundTime) - default: - return ErrUnsupportedPartitionStrategy - } - - partitionAttached, err := p.isPartitionIsAttached(partition) - if err != nil { - return fmt.Errorf("failed to check partition attachment status: %w", err) - } - - if partitionAttached { - p.logger.Info("Table is already attached to the parent table, skip", "schema", partition.Schema, "table", partition.Name) - - return nil - } - - maxRetries := 3 - - err = retry.WithRetry(maxRetries, func(attempt int) error { - err := p.attachPartition(partition) - if err != nil { - p.logger.Warn("fail to attach partition", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) - } - - return err - }) - if err != nil { - return fmt.Errorf("failed to attach partition after retries: %w", err) - } - - p.logger.Info("Partition attached to parent table", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.GetParentTable().Name) - - return nil -} - -func (p PostgreSQL) tableExists(table Table) (bool, error) { - query := `SELECT EXISTS( - SELECT c.oid - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 AND c.relname = $2 - );` - - var exists bool - - err := p.db.QueryRow(context.Background(), query, table.Schema, table.Name).Scan(&exists) - if err != nil { - return false, fmt.Errorf("failed to get table: %w", err) - } - - return exists, nil -} - -func (p PostgreSQL) GetPartitionSettings(table Table) (PartitionSettings, error) { - var partkeydef []string - - maxRetries := 3 - - err := retry.WithRetry(maxRetries, func(attempt int) error { - // pg_get_partkeydef() is a system function returning the definition of a partitioning key - // It return a text string: () - // Example for RANGE (created_at) - query := fmt.Sprintf(` - SELECT regexp_match(partkeydef, '^(.*) \((.*)\)$') - FROM pg_catalog.pg_get_partkeydef('%s'::regclass) as partkeydef - `, table.QualifiedName()) - - err := p.db.QueryRow(p.ctx, query).Scan(&partkeydef) - if err != nil { - p.logger.Warn("failed to get partitioning key", "error", err, "schema", table.Schema, "table", table.Name, "attempt", attempt, "max_retries", maxRetries) - - return fmt.Errorf("failed to get partition key: %w", err) - } - - return nil - }) - if err != nil { - return PartitionSettings{}, fmt.Errorf("failed to get partitioning key after retries: %w", err) - } - - if len(partkeydef) == 0 { - return PartitionSettings{}, ErrPartitionNotFound - } - - settings := PartitionSettings{ - Key: partkeydef[1], - } - - rawPartitionStrategy := partkeydef[0] - switch rawPartitionStrategy { - case string(RangePartitionStrategy): - settings.Strategy = RangePartitionStrategy - case string(ListPartitionStrategy): - settings.Strategy = ListPartitionStrategy - case string(HashPartitionStrategy): - settings.Strategy = HashPartitionStrategy - default: - return settings, fmt.Errorf("%w: %s", ErrUnsupportedPartitionStrategy, rawPartitionStrategy) - } - - keyColumn := Column{ - Schema: table.Schema, - Table: table.Name, - Name: settings.Key, - } - - settings.KeyType, err = p.getColumnDataType(keyColumn) - if err != nil { - return settings, fmt.Errorf("failed to get partition key data type: %s: %w", rawPartitionStrategy, err) - } - - return settings, nil -} - -func (p PostgreSQL) isPartitionIsAttached(partition Partition) (bool, error) { - query := `SELECT EXISTS( - SELECT 1 FROM pg_inherits WHERE inhrelid = $1::regclass - )` - - var exists bool - - err := p.db.QueryRow(context.Background(), query, partition.QualifiedName()).Scan(&exists) - if err != nil { - return false, fmt.Errorf("failed to check partition attachment: %w", err) - } - - return exists, nil -} - -func (p PostgreSQL) attachPartition(partition Partition) error { - query := fmt.Sprintf("ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM ('%s') TO ('%s')", partition.GetParentTable().QualifiedName(), partition.QualifiedName(), partition.LowerBound, partition.UpperBound) - p.logger.Debug("Attach partition", "query", query, "partition", partition.Name, "table", partition.GetParentTable().Name) - - _, err := p.db.Exec(context.Background(), query) - if err != nil { - return fmt.Errorf("failed to attach partition: %w", err) - } - - return nil -} - -// Function detachPartitionConcurrently detaches specified partition from the parent table. -// The partition exists as standalone table after detaching. -// More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION -func (p PostgreSQL) detachPartitionConcurrently(partition Partition) error { - query := fmt.Sprintf(`ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY;`, partition.GetParentTable().Name, partition.QualifiedName()) - p.logger.Debug("Detach partition", "schema", partition.Schema, "table", partition.Name, "query", query, "parent_table", partition.GetParentTable().Name) - - _, err := p.db.Exec(context.Background(), query) - if err != nil { - return fmt.Errorf("failed to detach partition from the parent table: %w", err) - } - - return nil -} - -// Function finalizePartitionDetach finalize a partition detach operation. -// It's required when a partition is in "detach pending" status. -// More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION -func (p PostgreSQL) finalizePartitionDetach(partition Partition) error { - query := fmt.Sprintf(`ALTER TABLE %s DETACH PARTITION %s FINALIZE;`, partition.GetParentTable().Name, partition.QualifiedName()) - p.logger.Debug("finialize detach partition", "schema", partition.Schema, "table", partition.Name, "query", query, "parent_table", partition.GetParentTable().Name) - - _, err := p.db.Exec(context.Background(), query) - if err != nil { - return fmt.Errorf("failed to finalize partition detach: %w", err) - } - - return nil -} - -func (p PostgreSQL) dropTable(table Table) error { - query := fmt.Sprintf("DROP TABLE %s", table.QualifiedName()) - p.logger.Debug("Drop table", "schema", table.Schema, "table", table.Name, "query", query) - - _, err := p.db.Exec(context.Background(), query) - if err != nil { - return fmt.Errorf("failed to drop table: %w", err) - } - - return nil -} - -func (p PostgreSQL) ListPartitions(table Table) (partitions []Partition, err error) { - query := fmt.Sprintf(` - WITH parts as ( - SELECT - relnamespace::regnamespace as schema, - c.oid::pg_catalog.regclass AS part_name, - regexp_match(pg_get_expr(c.relpartbound, c.oid), - 'FOR VALUES FROM \(''(.*)''\) TO \(''(.*)''\)') AS bounds - FROM - pg_catalog.pg_class c JOIN pg_catalog.pg_inherits i ON (c.oid = i.inhrelid) - WHERE i.inhparent = '%s'::regclass - AND c.relkind='r' - ) - SELECT - schema, - part_name as name, - '%s' as parentTable, - bounds[1]::text AS lower_bound, - bounds[2]::text AS upper_bound - FROM parts - ORDER BY part_name;`, table.QualifiedName(), table.Name) - - rows, err := p.db.Query(context.Background(), query) - if err != nil { - return nil, fmt.Errorf("failed to list partitions: %w", err) - } - - rawPartitions, err := pgx.CollectRows(rows, pgx.RowToStructByName[Partition]) - if err != nil { - return nil, fmt.Errorf("failed to cast list: %w", err) - } - - for _, partition := range rawPartitions { - lowerBound, upperBound, err := parseBounds(partition) - if err != nil { - return nil, fmt.Errorf("failed to parse bounds: %w", err) - } - - partition.LowerBound = lowerBound - partition.UpperBound = upperBound - - partitions = append(partitions, partition) - } - - return partitions, nil -} - -func (p PostgreSQL) DetachPartition(partition Partition) error { - p.logger.Debug("Detach partition", "schema", partition.Schema, "table", partition.Name) - - maxRetries := 3 - - err := retry.WithRetry(maxRetries, func(attempt int) error { - err := p.detachPartitionConcurrently(partition) - if err != nil { - // detachPartitionConcurrently() could fail if the specified partition is in pending detach status - // It could occurred when a previous detach partition concurrently operation was canceled or interrupted - // It prevent any other detach operations on the table - // More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION - // To unblock the situation, we try to finalize the detach operation on Object Not In Prerequisite State error - if isPostgreSQLErrorCode(err, ObjectNotInPrerequisiteStatePostgreSQLErrorCode) { - p.logger.Warn("Table is already pending detach in partitioned, retry with finalize", "error", err, "schema", partition.Schema, "table", partition.Name) - - finalizeErr := p.finalizePartitionDetach(partition) - if finalizeErr == nil { - err = nil // Returns a success since the partition detach operation has been completed - } - } else { - p.logger.Warn("Fail to detach partition", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) - } - } - - return err - }) - if err != nil { - return fmt.Errorf("failed to detach partition after retries: %w", err) - } - - return nil -} - -func (p PostgreSQL) DeletePartition(partition Partition) error { - p.logger.Debug("Deleting partition", "schema", partition.Schema, "table", partition.Name) - - maxRetries := 3 - - err := retry.WithRetry(maxRetries, func(attempt int) error { - err := p.dropTable(partition.ToTable()) - if err != nil { - p.logger.Warn("Fail to drop table", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) - } - - return err - }) - if err != nil { - return fmt.Errorf("failed to drop table after retries: %w", err) - } - - return nil -} diff --git a/internal/infra/postgresql/postgresql_internal_test.go b/internal/infra/postgresql/postgresql_internal_test.go deleted file mode 100644 index 8f9d929..0000000 --- a/internal/infra/postgresql/postgresql_internal_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package postgresql - -import ( - "fmt" - "testing" - - "github.com/pashagolub/pgxmock/v3" - "github.com/qonto/postgresql-partition-manager/internal/infra/logger" - "github.com/stretchr/testify/assert" -) - -func getMock(t *testing.T) (pgxmock.PgxConnIface, *PostgreSQL) { - t.Helper() - - mock, err := pgxmock.NewConn() - if err != nil { - fmt.Println("ERROR: Fail to initialize PostgreSQL mock: %w", err) - panic(err) - } - - logger, err := logger.New(false, "text") - if err != nil { - fmt.Println("ERROR: Fail to initialize logger: %w", err) - panic(err) - } - - p := New(*logger, mock) - - return mock, p -} - -func TestTableExists(t *testing.T) { - mock, p := getMock(t) - - existingTable := Table{Schema: "public", Name: "my_table"} - existingTables := []Table{existingTable} - - query := `SELECT EXISTS\( SELECT c.oid FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = \$1 AND c.relname = \$2 \)` - for _, table := range existingTables { - mock.ExpectQuery(query).WithArgs(table.Schema, table.Name).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(true)) - } - - tableWithSameNameInDifferentSchema := Table{Schema: "another_schema", Name: "my_table"} - tableWithSameSchemaButDifferentName := Table{Schema: "public", Name: "another_table"} - missingTables := []Table{tableWithSameNameInDifferentSchema, tableWithSameSchemaButDifferentName} - - for _, table := range missingTables { - mock.ExpectQuery(query).WithArgs(table.Schema, table.Name).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(false)) - } - - testCases := []struct { - name string - table Table - exists bool - }{ - { - "Existing table", - existingTable, - true, - }, - { - "Table with same name in different schema", - tableWithSameNameInDifferentSchema, - false, - }, - { - "Table with same schema, but different name", - tableWithSameSchemaButDifferentName, - false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - exists, err := p.tableExists(tc.table) - - assert.Nil(t, err, "tableExists should succeed") - assert.Equal(t, exists, tc.exists, "Exists should match") - }) - } -} - -func TestIsPartitionIsAttached(t *testing.T) { - mock, p := getMock(t) - - attachedPartition := Partition{ParentTable: "partioned_table", Schema: "public", Name: "partioned_table_2024"} - unattachedPartition := Partition{ParentTable: "partioned_table", Schema: "public", Name: "partioned_table_1999"} - - query := `SELECT EXISTS\( SELECT 1 FROM pg_inherits WHERE inhrelid = \$1::regclass \)` - mock.ExpectQuery(query).WithArgs(attachedPartition.QualifiedName()).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(true)) - mock.ExpectQuery(query).WithArgs(unattachedPartition.QualifiedName()).WillReturnRows(mock.NewRows([]string{"exists"}).AddRow(false)) - - testCases := []struct { - name string - partition Partition - exists bool - }{ - { - "Attached partition", - attachedPartition, - true, - }, - { - "Unattached partition", - unattachedPartition, - false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - exists, err := p.isPartitionIsAttached(tc.partition) - - assert.Nil(t, err, "isPartitionIsAttached should succeed") - assert.Equal(t, exists, tc.exists, "Exists should match") - }) - } -} - -func TestDropTable(t *testing.T) { - mock, p := getMock(t) - - table := Table{Schema: "public", Name: "my_table"} - - query := fmt.Sprintf("DROP TABLE %s", table.QualifiedName()) - mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("DROP", 1)) - - err := p.dropTable(table) - assert.Nil(t, err, "dropTable should succeed") -} - -func TestGetPartitionSettings(t *testing.T) { - mock, p := getMock(t) - - queryPartitionSettings := `SELECT regexp_match\(partkeydef` // Partial query - queryColumn := `SELECT data_type as columnType FROM information_schema.columns WHERE table_schema = \$1 AND table_name = \$2 AND column_name = \$3` - - testCases := []struct { - name string - column Column - expectedSettings PartitionSettings - }{ - { - "Date partition", - Column{ - Schema: "public", - Table: "my_table", - Name: "created_at", - DataType: DateColumnType, - }, - PartitionSettings{ - Strategy: RangePartitionStrategy, - Key: "created_at", - KeyType: DateColumnType, - }, - }, - { - "Datetime partition", - Column{ - Schema: "public", - Table: "my_table", - Name: "hour", - DataType: DateTimeColumnType, - }, - PartitionSettings{ - Strategy: RangePartitionStrategy, - Key: "hour", - KeyType: DateTimeColumnType, - }, - }, - { - "UUID partition", - Column{ - Schema: "public", - Table: "my_table", - Name: "id", - DataType: UUIDColumnType, - }, - PartitionSettings{ - Strategy: RangePartitionStrategy, - Key: "id", - KeyType: UUIDColumnType, - }, - }, - { - "List partition", - Column{ - Schema: "public", - Table: "my_table", - Name: "created_at", - DataType: UUIDColumnType, - }, - PartitionSettings{ - Strategy: ListPartitionStrategy, - Key: "created_at", - KeyType: UUIDColumnType, - }, - }, - { - "Hash partition", - Column{ - Schema: "public", - Table: "my_table", - Name: "created_at", - DataType: UUIDColumnType, - }, - PartitionSettings{ - Strategy: HashPartitionStrategy, - Key: "created_at", - KeyType: UUIDColumnType, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"partkeydef"}).AddRow([]string{string(tc.expectedSettings.Strategy), tc.column.Name})) - mock.ExpectQuery(queryColumn).WithArgs(tc.column.Schema, tc.column.Table, tc.column.Name).WillReturnRows(mock.NewRows([]string{"columnType"}).AddRow(string(tc.column.DataType))) - - table := Table{Schema: tc.column.Schema, Name: tc.column.Table} - settings, err := p.GetPartitionSettings(table) - - assert.Nil(t, err, "GetPartitionSettings should succeed") - assert.Equal(t, settings, tc.expectedSettings, "partition settings should succeed") - }) - } -} - -func TestGetPartitionSettingsErrors(t *testing.T) { - mock, p := getMock(t) - - queryPartitionSettings := `SELECT regexp_match\(partkeydef` // Partial query - queryColumn := `SELECT data_type as columnType FROM information_schema.columns WHERE table_schema = \$1 AND table_name = \$2 AND column_name = \$3` - - testCases := []struct { - name string - column Column - expectedSettings PartitionSettings - }{ - { - "Missing partition", - Column{ - Schema: "public", - Table: "my_table", - Name: "created_at", - DataType: DateColumnType, - }, - PartitionSettings{ - Strategy: RangePartitionStrategy, - Key: "created_at", - KeyType: DateColumnType, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - maxRetries := 3 - for attempt := 1; attempt <= maxRetries; attempt++ { - mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"x"})) - mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"x"})) - mock.ExpectQuery(queryPartitionSettings).WillReturnRows(mock.NewRows([]string{"x"})) - } - mock.ExpectQuery(queryColumn).WithArgs(tc.column.Schema, tc.column.Table, tc.column.Name).WillReturnRows(mock.NewRows([]string{"x"})) - - table := Table{Schema: tc.column.Schema, Name: tc.column.Table} - _, err := p.GetPartitionSettings(table) - - assert.Error(t, err, ErrPartitionNotFound) - }) - } -} diff --git a/internal/infra/postgresql/postgresql_test.go b/internal/infra/postgresql/postgresql_test.go new file mode 100644 index 0000000..d460f4c --- /dev/null +++ b/internal/infra/postgresql/postgresql_test.go @@ -0,0 +1,35 @@ +package postgresql_test + +import ( + "context" + "testing" + + "github.com/jackc/pgconn" + "github.com/pashagolub/pgxmock/v3" + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +var ErrPostgreSQLConnectionFailure = &pgconn.PgError{ + Code: "08006", +} + +func setupMock(t *testing.T, queryMatcher pgxmock.QueryMatcher) (pgxmock.PgxConnIface, *postgresql.Postgres) { + t.Helper() + + mock, err := pgxmock.NewConn(pgxmock.QueryMatcherOption(queryMatcher)) + if err != nil { + t.Fatalf("ERROR: Fail to initialize PostgreSQL mock: %s", err) + } + + defer mock.Close(context.TODO()) //nolint:golint,errcheck + + logger, err := logger.New(false, "text") + if err != nil { + t.Fatalf("ERROR: Fail to initialize logger: %s", err) + } + + client := postgresql.New(*logger, mock) + + return mock, client +} diff --git a/internal/infra/postgresql/server.go b/internal/infra/postgresql/server.go index 60f3411..d8a7054 100644 --- a/internal/infra/postgresql/server.go +++ b/internal/infra/postgresql/server.go @@ -11,10 +11,15 @@ import ( var ErrUnkownServerVersion = errors.New("could not find server version") -func (p *PostgreSQL) GetVersion() (int64, error) { - serverVersionStr := p.db.PgConn().ParameterStatus("server_version") - serverVersionStr = regexp.MustCompile(`^[0-9]+`).FindString(serverVersionStr) +func (p *Postgres) GetEngineVersion() (int64, error) { + var serverVersionRaw string + err := p.conn.QueryRow(context.Background(), "SHOW server_version").Scan(&serverVersionRaw) + if err != nil { + return 0, fmt.Errorf("could not get server version: %w", err) + } + + serverVersionStr := regexp.MustCompile(`^[0-9]+`).FindString(serverVersionRaw) if serverVersionStr == "" { return 0, ErrUnkownServerVersion } @@ -27,10 +32,8 @@ func (p *PostgreSQL) GetVersion() (int64, error) { return serverVersion, nil } -func (p *PostgreSQL) GetServerTime() (time.Time, error) { - var serverTime time.Time - - err := p.db.QueryRow(context.Background(), "SELECT NOW() AT TIME ZONE 'UTC' as serverTime").Scan(&serverTime) +func (p *Postgres) GetServerTime() (serverTime time.Time, err error) { + err = p.conn.QueryRow(context.Background(), "SELECT NOW() AT TIME ZONE 'UTC' as serverTime").Scan(&serverTime) if err != nil { return time.Time{}, fmt.Errorf("could not get server time: %w", err) } diff --git a/internal/infra/postgresql/server_test.go b/internal/infra/postgresql/server_test.go index 7f4a0f1..3a64bc8 100644 --- a/internal/infra/postgresql/server_test.go +++ b/internal/infra/postgresql/server_test.go @@ -1,36 +1,63 @@ +//nolint:golint,wsl package postgresql_test import ( - "fmt" "testing" "time" "github.com/pashagolub/pgxmock/v3" - "github.com/qonto/postgresql-partition-manager/internal/infra/logger" - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" "github.com/stretchr/testify/assert" ) func TestGetServerTime(t *testing.T) { - mock, err := pgxmock.NewConn() - if err != nil { - fmt.Println("ERROR: Fail to initialize PostgreSQL mock: %w", err) - panic(err) - } + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + query := `SELECT NOW() AT TIME ZONE 'UTC' as serverTime` currrentTime := time.Now() - query := `SELECT NOW\(\) AT TIME ZONE \'UTC\' as serverTime` + mock.ExpectQuery(query).WillReturnRows(mock.NewRows([]string{"serverTime"}).AddRow(currrentTime)) + serverTime, err := p.GetServerTime() + assert.Nil(t, err, "GetServerTime should succeed") + assert.Equal(t, serverTime, currrentTime, "Time should match") + + mock.ExpectQuery(query).WillReturnError(ErrPostgreSQLConnectionFailure) + _, err = p.GetServerTime() + assert.Error(t, err, "GetServerTime should fail") +} - logger, err := logger.New(false, "text") - if err != nil { - fmt.Println("ERROR: Fail to initialize logger: %w", err) - panic(err) +func TestGetEngineVersion(t *testing.T) { + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + query := `SHOW server_version` + + testCases := []struct { + output string + version int64 + }{ + { + "16.2 (Debian 16.2-1.pgdg120+2)", + 16, + }, + { + "14.1", + 14, + }, } - p := postgresql.New(*logger, mock) - serverTime, err := p.GetServerTime() + for _, tc := range testCases { + t.Run(tc.output, func(t *testing.T) { + mock.ExpectQuery(query).WillReturnRows(mock.NewRows([]string{"server_version"}).AddRow(tc.output)) - assert.Nil(t, err, "GetServerTime should succeed") - assert.Equal(t, serverTime, currrentTime, "Time should match") + version, err := p.GetEngineVersion() + assert.Nil(t, err, "GetEngineVersion should succeed") + assert.Equal(t, version, tc.version, "Version mismatch") + }) + } + + mock.ExpectQuery(query).WillReturnError(ErrPostgreSQLConnectionFailure) + _, err := p.GetEngineVersion() + assert.Error(t, err, "GetEngineVersion should fail") + + mock.ExpectQuery(query).WillReturnRows(mock.NewRows([]string{"server_version"}).AddRow("unexpected version format")) + _, err = p.GetEngineVersion() + assert.Error(t, err, "GetEngineVersion should fail") } diff --git a/internal/infra/postgresql/table.go b/internal/infra/postgresql/table.go index 40c4699..2f78f85 100644 --- a/internal/infra/postgresql/table.go +++ b/internal/infra/postgresql/table.go @@ -2,17 +2,42 @@ package postgresql import "fmt" -type Table struct { - Schema string - Name string +func (p Postgres) CreateTableLikeTable(schema, table, parent string) error { + query := fmt.Sprintf("CREATE TABLE %s.%s (LIKE %s.%s)", schema, table, schema, parent) + p.logger.Debug("Create table", "query", schema, "table", table, "query", query) + + _, err := p.conn.Exec(p.ctx, query) + if err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + return nil } -func (t Table) String() string { - return t.QualifiedName() +func (p Postgres) DropTable(schema, table string) error { + query := fmt.Sprintf("DROP TABLE %s.%s", schema, table) + p.logger.Debug("Drop table", "schema", schema, "table", table, "query", query) + + _, err := p.conn.Exec(p.ctx, query) + if err != nil { + return fmt.Errorf("failed to drop table: %w", err) + } + + return nil } -// QualifiedName returns the fully qualified name of the table (format: .
) -// This is recommended to avoid schema conflicts when querying PostgreSQL catalog tables -func (t Table) QualifiedName() string { - return fmt.Sprintf("%s.%s", t.Schema, t.Name) +func (p Postgres) IsTableExists(schema, table string) (exists bool, err error) { + query := `SELECT EXISTS( + SELECT c.oid + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 + );` + + err = p.conn.QueryRow(p.ctx, query, schema, table).Scan(&exists) + if err != nil { + return false, fmt.Errorf("failed to check if table exists: %w", err) + } + + return exists, nil } diff --git a/internal/infra/postgresql/table_test.go b/internal/infra/postgresql/table_test.go index 08ee7e6..4f183b0 100644 --- a/internal/infra/postgresql/table_test.go +++ b/internal/infra/postgresql/table_test.go @@ -1,32 +1,68 @@ +//nolint:golint,wsl package postgresql_test import ( + "fmt" "testing" - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) -func TestTableAttributes(t *testing.T) { - testCases := []struct { - name string - table postgresql.Table - expectedName string - }{ - { - name: "Public schema", - table: postgresql.Table{ - Schema: "public", - Name: "my_table", - }, - expectedName: "public.my_table", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.table.QualifiedName(), tc.expectedName, "Table name name don't match") - assert.Equal(t, tc.table.String(), tc.expectedName, "Table name name don't match") - }) - } +func TestCreateTableLikeTable(t *testing.T) { + schema := "public" + table := "my_table" + parentTable := "parent_table" + + query := fmt.Sprintf(`CREATE TABLE %s.%s (LIKE %s.%s)`, schema, table, schema, parentTable) + + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + + mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("CREATE", 1)) + err := p.CreateTableLikeTable(schema, table, parentTable) + assert.Nil(t, err, "DropTable should succeed") + + mock.ExpectExec(query).WillReturnError(ErrPostgreSQLConnectionFailure) + err = p.CreateTableLikeTable(schema, table, parentTable) + assert.Error(t, err, "DropTable should fail") +} + +func TestDropTable(t *testing.T) { + schema := "public" + table := "my_table" + fullQualifiedName := fmt.Sprintf("%s.%s", schema, table) + query := fmt.Sprintf(`DROP TABLE %s`, fullQualifiedName) + + mock, p := setupMock(t, pgxmock.QueryMatcherEqual) + + mock.ExpectExec(query).WillReturnResult(pgxmock.NewResult("DROP", 1)) + err := p.DropTable(schema, table) + assert.Nil(t, err, "DropTable should succeed") + + mock.ExpectExec(query).WillReturnError(ErrPostgreSQLConnectionFailure) + err = p.DropTable(schema, table) + assert.Error(t, err, "DropTable should fail") +} + +func TestIsTableExists(t *testing.T) { + schema := "public" + table := "my_table" + + query := "SELECT EXISTS" + + mock, p := setupMock(t, pgxmock.QueryMatcherRegexp) + + mock.ExpectQuery(query).WithArgs(schema, table).WillReturnRows(mock.NewRows([]string{"EXISTS"}).AddRow(true)) + exists, err := p.IsTableExists(schema, table) + assert.Nil(t, err, "IsTableExists should succeed") + assert.True(t, exists, "Table should exists") + + mock.ExpectQuery(query).WithArgs(schema, table).WillReturnRows(mock.NewRows([]string{"EXISTS"}).AddRow(false)) + exists, err = p.IsTableExists(schema, table) + assert.Nil(t, err, "IsTableExists should succeed") + assert.False(t, exists, "Table should not exists") + + mock.ExpectQuery(query).WillReturnError(ErrPostgreSQLConnectionFailure) + _, err = p.IsTableExists(schema, table) + assert.Error(t, err, "IsTableExists should fail") } diff --git a/pkg/ppm/server_test.go b/pkg/ppm/server_test.go index ca64807..1f538bb 100644 --- a/pkg/ppm/server_test.go +++ b/pkg/ppm/server_test.go @@ -51,7 +51,7 @@ func TestServerRequirements(t *testing.T) { logger, postgreSQLMock := getTestMocks(t) checker := ppm.New(context.TODO(), *logger, postgreSQLMock, nil) - postgreSQLMock.On("GetVersion").Return(tc.serverVersion, nil).Once() + postgreSQLMock.On("GetEngineVersion").Return(tc.serverVersion, nil).Once() postgreSQLMock.On("GetServerTime").Return(tc.serverTime, nil).Once() err := checker.CheckServerRequirements() From da95adb1c3db594d71b2eaf65a628cf31d28c836 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Mon, 22 Apr 2024 18:19:56 +0200 Subject: [PATCH 21/27] refact(partition): Rewrite package to split PostgreSQL logic --- internal/infra/config/config.go | 14 +-- internal/infra/partition/bounds.go | 50 ++++++++ internal/infra/partition/configuration.go | 141 ++++++++++++++++++++++ internal/infra/partition/error.go | 5 + internal/infra/partition/interval.go | 12 ++ internal/infra/partition/partition.go | 25 ++++ internal/infra/partition/strategy.go | 9 ++ 7 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 internal/infra/partition/bounds.go create mode 100644 internal/infra/partition/configuration.go create mode 100644 internal/infra/partition/error.go create mode 100644 internal/infra/partition/interval.go create mode 100644 internal/infra/partition/partition.go create mode 100644 internal/infra/partition/strategy.go diff --git a/internal/infra/config/config.go b/internal/infra/config/config.go index 73f8ff7..de92acc 100644 --- a/internal/infra/config/config.go +++ b/internal/infra/config/config.go @@ -6,16 +6,16 @@ import ( "fmt" "github.com/go-playground/validator/v10" - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" ) type Config struct { - Debug bool `mapstructure:"debug"` - LogFormat string `mapstructure:"log-format"` - ConnectionURL string `mapstructure:"connection-url"` - StatementTimeout int `mapstructure:"statement-timeout" validate:"required"` - LockTimeout int `mapstructure:"lock-timeout" validate:"required"` - Partitions map[string]postgresql.PartitionConfiguration `mapstructure:"partitions" validate:"required,dive,keys,endkeys,required"` + Debug bool `mapstructure:"debug"` + LogFormat string `mapstructure:"log-format"` + ConnectionURL string `mapstructure:"connection-url"` + StatementTimeout int `mapstructure:"statement-timeout" validate:"required"` + LockTimeout int `mapstructure:"lock-timeout" validate:"required"` + Partitions map[string]partition.Configuration `mapstructure:"partitions" validate:"required,dive,keys,endkeys,required"` } func (c *Config) Check() error { diff --git a/internal/infra/partition/bounds.go b/internal/infra/partition/bounds.go new file mode 100644 index 0000000..dc998c4 --- /dev/null +++ b/internal/infra/partition/bounds.go @@ -0,0 +1,50 @@ +package partition + +import ( + "errors" + "time" + + "github.com/google/uuid" +) + +const ( + UUIDv7Version uuid.Version = 7 + nbDaysInAWeek int = 7 +) + +var ( + ErrLowerBoundAfterUpperBound = errors.New("lowerbound is after upperbound") + + // ErrCantDecodePartitionBounds represents an error indicating that the partition bounds cannot be decoded. + ErrCantDecodePartitionBounds = errors.New("partition bounds cannot be decoded") + + ErrUnsupportedUUIDVersion = errors.New("unsupported UUID version") +) + +func getDailyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, date.UTC().Location()) + upperBound = lowerBound.AddDate(0, 0, 1) + + return +} + +func getWeeklyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, date.UTC().Location()).AddDate(0, 0, -int(date.Weekday()-time.Monday)) + upperBound = lowerBound.AddDate(0, 0, nbDaysInAWeek) + + return +} + +func getMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, date.UTC().Location()) + upperBound = lowerBound.AddDate(0, 1, 0) + + return +} + +func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) { + lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, date.UTC().Location()) + upperBound = lowerBound.AddDate(1, 0, 0) + + return +} diff --git a/internal/infra/partition/configuration.go b/internal/infra/partition/configuration.go new file mode 100644 index 0000000..ed7fd4a --- /dev/null +++ b/internal/infra/partition/configuration.go @@ -0,0 +1,141 @@ +package partition + +import ( + "fmt" + "time" +) + +type ( + CleanupPolicy string +) + +const ( + Drop CleanupPolicy = "drop" + Detach CleanupPolicy = "detach" +) + +type Configuration struct { + Schema string `mapstructure:"schema" validate:"required"` + Table string `mapstructure:"table" validate:"required"` + PartitionKey string `mapstructure:"partitionKey" validate:"required"` + Interval Interval `mapstructure:"interval" validate:"required,oneof=daily weekly monthly yearly"` + Retention int `mapstructure:"retention" validate:"required,gt=0"` + PreProvisioned int `mapstructure:"preProvisioned" validate:"required,gt=0"` + CleanupPolicy CleanupPolicy `mapstructure:"cleanupPolicy" validate:"required,oneof=drop detach"` +} + +func (p Configuration) GeneratePartition(forDate time.Time) (Partition, error) { + var suffix string + + var lowerBound, upperBound time.Time + + switch p.Interval { + case Daily: + suffix = forDate.Format("2006_01_02") + lowerBound, upperBound = getDailyBounds(forDate) + case Weekly: + year, week := forDate.ISOWeek() + suffix = fmt.Sprintf("%d_w%02d", year, week) + lowerBound, upperBound = getWeeklyBounds(forDate) + case Monthly: + suffix = forDate.Format("2006_01") + lowerBound, upperBound = getMonthlyBounds(forDate) + case Yearly: + suffix = forDate.Format("2006") + lowerBound, upperBound = getYearlyBounds(forDate) + default: + return Partition{}, ErrUnsupportedInterval + } + + partition := Partition{ + Schema: p.Schema, + ParentTable: p.Table, + Name: fmt.Sprintf("%s_%s", p.Table, suffix), + LowerBound: lowerBound, + UpperBound: upperBound, + } + + return partition, nil +} + +func (p Configuration) GetRetentionPartitions(forDate time.Time) ([]Partition, error) { + partitions := make([]Partition, p.Retention) + + for i := 1; i <= p.Retention; i++ { + prevDate, err := p.getPrevDate(forDate, i) + if err != nil { + return nil, fmt.Errorf("could not compute previous date: %w", err) + } + + partition, err := p.GeneratePartition(prevDate) + if err != nil { + return nil, fmt.Errorf("could not generate partition: %w", err) + } + + partitions[i-1] = partition + } + + return partitions, nil +} + +func (p Configuration) GetPreProvisionedPartitions(forDate time.Time) ([]Partition, error) { + partitions := make([]Partition, p.PreProvisioned) + + for i := 1; i <= p.PreProvisioned; i++ { + nextDate, err := p.getNextDate(forDate, i) + if err != nil { + return nil, fmt.Errorf("could not compute next date: %w", err) + } + + partition, err := p.GeneratePartition(nextDate) + if err != nil { + return nil, fmt.Errorf("could not generate partition: %w", err) + } + + partitions[i-1] = partition + } + + return partitions, nil +} + +func (p Configuration) getPrevDate(forDate time.Time, i int) (t time.Time, err error) { + switch p.Interval { + case Daily: + t = forDate.AddDate(0, 0, -i) + case Weekly: + t = forDate.AddDate(0, 0, -i*nbDaysInAWeek) + case Monthly: + year, month, _ := forDate.Date() + + t = time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, forDate.Location()) + case Yearly: + year, _, _ := forDate.Date() + + t = time.Date(year-i, 1, 1, 0, 0, 0, 0, forDate.Location()) + default: + return time.Time{}, ErrUnsupportedInterval + } + + return t, nil +} + +func (p Configuration) getNextDate(forDate time.Time, i int) (t time.Time, err error) { + switch p.Interval { + case Daily: + t = forDate.AddDate(0, 0, i) + case Weekly: + t = forDate.AddDate(0, 0, i*nbDaysInAWeek) + case Monthly: + year, month, _ := forDate.Date() + + t = time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, forDate.Location()) + case Yearly: + year, _, _ := forDate.Date() + + t = time.Date(year+i, 1, 1, 0, 0, 0, 0, forDate.Location()) + default: + return time.Time{}, ErrUnsupportedInterval + } + + return t, nil +} diff --git a/internal/infra/partition/error.go b/internal/infra/partition/error.go new file mode 100644 index 0000000..a2e47e1 --- /dev/null +++ b/internal/infra/partition/error.go @@ -0,0 +1,5 @@ +package partition + +import "errors" + +var ErrUnsupportedInterval = errors.New("unsupported partition interval") diff --git a/internal/infra/partition/interval.go b/internal/infra/partition/interval.go new file mode 100644 index 0000000..30229b0 --- /dev/null +++ b/internal/infra/partition/interval.go @@ -0,0 +1,12 @@ +package partition + +type ( + Interval string +) + +const ( + Daily Interval = "daily" + Weekly Interval = "weekly" + Monthly Interval = "monthly" + Yearly Interval = "yearly" +) diff --git a/internal/infra/partition/partition.go b/internal/infra/partition/partition.go new file mode 100644 index 0000000..6ff2d6e --- /dev/null +++ b/internal/infra/partition/partition.go @@ -0,0 +1,25 @@ +// Package partition provides methods for PostgreSQL partition configuration +package partition + +import ( + "fmt" + "time" +) + +type Partition struct { + ParentTable string + Schema string + Name string + LowerBound time.Time + UpperBound time.Time +} + +func (p Partition) String() string { + return p.QualifiedName() +} + +// QualifiedName returns the fully qualified name of the partition (format: .
) +// This is recommended to avoid schema conflicts when querying PostgreSQL catalog tables +func (p Partition) QualifiedName() string { + return fmt.Sprintf("%s.%s", p.Schema, p.Name) +} diff --git a/internal/infra/partition/strategy.go b/internal/infra/partition/strategy.go new file mode 100644 index 0000000..3d8250e --- /dev/null +++ b/internal/infra/partition/strategy.go @@ -0,0 +1,9 @@ +package partition + +const ( + Range PartitionStrategy = "RANGE" + List PartitionStrategy = "LIST" + Hash PartitionStrategy = "HASH" +) + +type PartitionStrategy string From dcaf1c649448cf1fb8c951b88ddb0913e1d92c12 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Mon, 22 Apr 2024 18:19:29 +0200 Subject: [PATCH 22/27] refact(ppm): Rewrite package to split PostgreSQL logic --- pkg/ppm/bounds.go | 92 ++++++++ pkg/ppm/bounds_internal_test.go | 129 +++++++++++ pkg/ppm/check_test.go | 198 ----------------- pkg/ppm/{check.go => checkpartition.go} | 80 +++++-- pkg/ppm/checkpartition_test.go | 200 +++++++++++++++++ pkg/ppm/{server.go => checkserver.go} | 2 +- .../{server_test.go => checkserver_test.go} | 2 +- pkg/ppm/cleanup.go | 77 ++++++- pkg/ppm/cleanup_test.go | 87 +++++--- pkg/ppm/error.go | 17 ++ pkg/ppm/error_internal_test.go | 45 ++++ pkg/ppm/mocks/PostgreSQLClient.go | 206 +++++++++++++----- pkg/ppm/ppm.go | 24 +- pkg/ppm/ppm_test.go | 58 +++++ pkg/ppm/provisioning.go | 86 +++++++- pkg/ppm/provisioning_test.go | 52 +++-- 16 files changed, 1013 insertions(+), 342 deletions(-) create mode 100644 pkg/ppm/bounds.go create mode 100644 pkg/ppm/bounds_internal_test.go delete mode 100644 pkg/ppm/check_test.go rename pkg/ppm/{check.go => checkpartition.go} (60%) create mode 100644 pkg/ppm/checkpartition_test.go rename pkg/ppm/{server.go => checkserver.go} (97%) rename pkg/ppm/{server_test.go => checkserver_test.go} (96%) create mode 100644 pkg/ppm/error.go create mode 100644 pkg/ppm/error_internal_test.go create mode 100644 pkg/ppm/ppm_test.go diff --git a/pkg/ppm/bounds.go b/pkg/ppm/bounds.go new file mode 100644 index 0000000..5adeeda --- /dev/null +++ b/pkg/ppm/bounds.go @@ -0,0 +1,92 @@ +package ppm + +import ( + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +const ( + UUIDv7Version uuid.Version = 7 +) + +var ( + ErrLowerBoundAfterUpperBound = errors.New("lowerbound is after upperbound") + ErrCantDecodePartitionBounds = errors.New("partition bounds cannot be decoded") + ErrUnsupportedUUIDVersion = errors.New("unsupported UUID version") +) + +func parseBounds(partition postgresql.PartitionResult) (lowerBound time.Time, upperBound time.Time, err error) { + lowerBound, upperBound, err = parseBoundAsDate(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsDateTime(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + lowerBound, upperBound, err = parseBoundAsUUIDv7(partition) + if err == nil { + return lowerBound, upperBound, nil + } + + if lowerBound.After(lowerBound) { + return time.Time{}, time.Time{}, ErrLowerBoundAfterUpperBound + } + + return time.Time{}, time.Time{}, ErrCantDecodePartitionBounds +} + +func parseBoundAsDate(partition postgresql.PartitionResult) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02", partition.LowerBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as date: %w", err) + } + + upperBound, err = time.Parse("2006-01-02", partition.UpperBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as date: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsDateTime(partition postgresql.PartitionResult) (lowerBound, upperBound time.Time, err error) { + lowerBound, err = time.Parse("2006-01-02 15:04:05", partition.LowerBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as datetime: %w", err) + } + + upperBound, err = time.Parse("2006-01-02 15:04:05", partition.UpperBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as datetime: %w", err) + } + + return lowerBound, upperBound, nil +} + +func parseBoundAsUUIDv7(partition postgresql.PartitionResult) (lowerBound, upperBound time.Time, err error) { + lowerBoundUUID, err := uuid.Parse(partition.LowerBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse lowerbound as UUID: %w", err) + } + + upperBoundUUID, err := uuid.Parse(partition.UpperBound) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("can't parse upperbound as UUID: %w", err) + } + + if upperBoundUUID.Version() != UUIDv7Version || lowerBoundUUID.Version() != UUIDv7Version { + return time.Time{}, time.Time{}, ErrUnsupportedUUIDVersion + } + + upperBound = time.Unix(upperBoundUUID.Time().UnixTime()).UTC() + lowerBound = time.Unix(lowerBoundUUID.Time().UnixTime()).UTC() + + return lowerBound, upperBound, nil +} diff --git a/pkg/ppm/bounds_internal_test.go b/pkg/ppm/bounds_internal_test.go new file mode 100644 index 0000000..ad103c0 --- /dev/null +++ b/pkg/ppm/bounds_internal_test.go @@ -0,0 +1,129 @@ +package ppm + +import ( + "testing" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "gotest.tools/assert" +) + +func TestParseBounds(t *testing.T) { + testCases := []struct { + name string + partition postgresql.PartitionResult + lowerbound string + upperBound string + }{ + { + "Date bounds", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "2025-03-02", + }, + "2024-01-01T00:00:00Z", + "2025-03-02T00:00:00Z", + }, + { + "Datetime bounds", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01 10:00:00", + UpperBound: "2025-02-03 12:53:00", + }, + "2024-01-01T10:00:00Z", + "2025-02-03T12:53:00Z", + }, + { + "UUIDv7 bounds", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + "2024-01-01T00:00:00Z", + "2024-01-02T00:00:00Z", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + expectedLowerbound, err := time.Parse(time.RFC3339, tc.lowerbound) + assert.NilError(t, err, "LowerBound parsing failed") + + expectedUpperBound, err := time.Parse(time.RFC3339, tc.upperBound) + assert.NilError(t, err, "Upperbound parsing failed") + + lowerBound, upperBound, err := parseBounds(tc.partition) + + assert.NilError(t, err, "Bounds parsing should succeed") + assert.Equal(t, lowerBound, expectedLowerbound, "LowerBound mismatch") + assert.Equal(t, upperBound, expectedUpperBound, "UpperBound mismatch") + }) + } +} + +func TestParseInvalidBounds(t *testing.T) { + testCases := []struct { + name string + partition postgresql.PartitionResult + }{ + { + "UUID v1 upper bound", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "018cc251-f400-7100-0000-000000000000", // UUIDv7: 2024-01-01 + UpperBound: "47568e76-fb49-11ee-b9c7-325096b39f47", // UUIDv1 + }, + }, + { + "UUID v1 lower bound", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "ad5dac7a-fb46-11ee-be67-325096b39f47", // UUIDv1 + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + }, + { + "Mix date format", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "2024-01-02 00:00:00", + }, + }, + { + "Mix date and UUIDv7", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + }, + { + "Mix date and UUIDv7", + postgresql.PartitionResult{ + Schema: "public", + Name: "my_table", + LowerBound: "2024-01-01", + UpperBound: "018cc778-5000-7100-0000-000000000000", // UUIDv7: 2024-01-02 + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := parseBounds(tc.partition) + + assert.ErrorContains(t, err, "partition bounds cannot be decoded") + }) + } +} diff --git a/pkg/ppm/check_test.go b/pkg/ppm/check_test.go deleted file mode 100644 index 5f4f8e0..0000000 --- a/pkg/ppm/check_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package ppm_test - -import ( - "context" - "fmt" - "log/slog" - "testing" - "time" - - "github.com/qonto/postgresql-partition-manager/internal/infra/logger" - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" - "github.com/qonto/postgresql-partition-manager/pkg/ppm" - "github.com/qonto/postgresql-partition-manager/pkg/ppm/mocks" - "gotest.tools/assert" -) - -var ( - dayBeforeYesterday = yesterday.AddDate(0, 0, -1) - yesterday = time.Now().AddDate(0, 0, -1) - today = time.Now() - tomorrow = time.Now().AddDate(0, 0, +1) - dayAfterTomorrow = tomorrow.AddDate(0, 0, +1) -) - -func getTestMocks(t *testing.T) (*slog.Logger, *mocks.PostgreSQLClient) { - t.Helper() - - logger, err := logger.New(false, "text") - if err != nil { - fmt.Println("ERROR: Fail to initialize logger: %w", err) - panic(err) - } - - postgreSQLMock := mocks.PostgreSQLClient{} - - return logger, &postgreSQLMock -} - -func TestCheckPartitions(t *testing.T) { - logger, postgreSQLMock := getTestMocks(t) - - partitions := map[string]postgresql.PartitionConfiguration{} - partitions["daily partition"] = postgresql.PartitionConfiguration{Schema: "app", Table: "daily_table1", PartitionKey: "column", Interval: postgresql.DailyInterval, Retention: 2, PreProvisioned: 2} - partitions["daily partition without retention"] = postgresql.PartitionConfiguration{Schema: "public", Table: "daily_table2", PartitionKey: "created_at", Interval: postgresql.DailyInterval, Retention: 0, PreProvisioned: 1} - partitions["daily partition without preprovisioned"] = postgresql.PartitionConfiguration{Schema: "public", Table: "daily_table3", PartitionKey: "column", Interval: postgresql.DailyInterval, Retention: 4, PreProvisioned: 0} - partitions["weekly partition"] = postgresql.PartitionConfiguration{Schema: "public", Table: "weekly_table", PartitionKey: "weekly", Interval: postgresql.WeeklyInterval, Retention: 2, PreProvisioned: 2} - partitions["monthly partition"] = postgresql.PartitionConfiguration{Schema: "public", Table: "monthly_table", PartitionKey: "month", Interval: postgresql.MonthlyInterval, Retention: 2, PreProvisioned: 2} - partitions["yearly partition"] = postgresql.PartitionConfiguration{Schema: "public", Table: "yearly_table", PartitionKey: "year", Interval: postgresql.YearlyInterval, Retention: 4, PreProvisioned: 4} - - // Build mock for each partitions - for _, p := range partitions { - settings := postgresql.PartitionSettings{ - KeyType: postgresql.DateColumnType, - Strategy: postgresql.RangePartitionStrategy, - Key: p.PartitionKey, - } - postgreSQLMock.On("GetPartitionSettings", postgresql.Table{Schema: p.Schema, Name: p.Table}).Return(settings, nil).Once() - - var tables []postgresql.Partition - - var partition postgresql.Partition - - for i := 0; i <= p.Retention; i++ { - switch p.Interval { - case postgresql.DailyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i)) - case postgresql.WeeklyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i*7)) - case postgresql.MonthlyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(0, -i, 0)) - case postgresql.YearlyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(-i, 0, 0)) - default: - t.Errorf("unuspported partition interval in retention table mock") - } - - tables = append(tables, partition) - } - - for i := 0; i <= p.PreProvisioned; i++ { - switch p.Interval { - case postgresql.DailyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i)) - case postgresql.WeeklyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i*7)) - case postgresql.MonthlyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(0, i, 0)) - case postgresql.YearlyInterval: - partition, _ = p.GeneratePartition(time.Now().AddDate(i, 0, 0)) - default: - t.Errorf("unuspported partition interval in preprovisonned table mock") - } - - tables = append(tables, partition) - } - postgreSQLMock.On("ListPartitions", postgresql.Table{Schema: p.Schema, Name: p.Table}).Return(tables, nil).Once() - } - - checker := ppm.New(context.TODO(), *logger, postgreSQLMock, partitions) - assert.NilError(t, checker.CheckPartitions(), "Partitions should succeed") -} - -func TestCheckMissingPartitions(t *testing.T) { - logger, postgreSQLMock := getTestMocks(t) - - partition := postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.DailyInterval, - Retention: 2, - PreProvisioned: 2, - } - - todayPartition, _ := partition.GeneratePartition(today) - yesterdayPartition, _ := partition.GeneratePartition(yesterday) - tomorrowPartition, _ := partition.GeneratePartition(tomorrow) - - testCases := []struct { - name string - tables []postgresql.Partition - }{ - { - "Missing Yesterday retention partition", - []postgresql.Partition{ - todayPartition, - yesterdayPartition, - }, - }, - { - "Missing Tomorrow partition", - []postgresql.Partition{ - todayPartition, - tomorrowPartition, - }, - }, - { - "Missing Today partition", - []postgresql.Partition{ - yesterdayPartition, - tomorrowPartition, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - settings := postgresql.PartitionSettings{ - KeyType: postgresql.DateColumnType, - Strategy: postgresql.RangePartitionStrategy, - Key: partition.PartitionKey, - } - postgreSQLMock.On("GetPartitionSettings", postgresql.Table{Schema: partition.Schema, Name: partition.Table}).Return(settings, nil).Once() - - fmt.Println("tc.tables", tc.tables) - postgreSQLMock.On("ListPartitions", postgresql.Table{Schema: partition.Schema, Name: partition.Table}).Return(tc.tables, nil).Once() - - checker := ppm.New(context.TODO(), *logger, postgreSQLMock, map[string]postgresql.PartitionConfiguration{"test": partition}) - assert.Error(t, checker.CheckPartitions(), "at least one partition contains an invalid configuration") - }) - } -} - -func TestUnsupportedPartitionsStrategy(t *testing.T) { - logger, postgreSQLMock := getTestMocks(t) - - partition := postgresql.PartitionConfiguration{ - Schema: "public", - Table: "my_table", - PartitionKey: "created_at", - Interval: postgresql.DailyInterval, - Retention: 2, - PreProvisioned: 2, - } - - testCases := []struct { - name string - settings postgresql.PartitionSettings - }{ - { - "Unsupported list partition strategy", - postgresql.PartitionSettings{Strategy: postgresql.ListPartitionStrategy, Key: "created_at", KeyType: postgresql.DateColumnType}, - }, - { - "Unsupported hash partition strategy", - postgresql.PartitionSettings{Strategy: postgresql.HashPartitionStrategy, Key: "created_at", KeyType: postgresql.DateColumnType}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - postgreSQLMock.On("GetPartitionSettings", postgresql.Table{Schema: partition.Schema, Name: partition.Table}).Return(tc.settings, nil).Once() - - checker := ppm.New(context.TODO(), *logger, postgreSQLMock, map[string]postgresql.PartitionConfiguration{"test": partition}) - assert.Error(t, checker.CheckPartitions(), "at least one partition contains an invalid configuration") - }) - } -} diff --git a/pkg/ppm/check.go b/pkg/ppm/checkpartition.go similarity index 60% rename from pkg/ppm/check.go rename to pkg/ppm/checkpartition.go index 731361f..30184c1 100644 --- a/pkg/ppm/check.go +++ b/pkg/ppm/checkpartition.go @@ -3,8 +3,10 @@ package ppm import ( "errors" "fmt" + "slices" "time" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" ) @@ -16,6 +18,12 @@ var ( ErrInvalidPartitionConfiguration = errors.New("at least one partition contains an invalid configuration") ) +var SupportedPartitionKeyDataType = []postgresql.ColumnType{ + postgresql.Date, + postgresql.DateTime, + postgresql.UUID, +} + func (p *PPM) CheckPartitions() error { partitionContainsAnError := false @@ -34,10 +42,12 @@ func (p *PPM) CheckPartitions() error { return ErrInvalidPartitionConfiguration } + p.logger.Info("All partitions are correctly configured") + return nil } -func (p *PPM) checkPartition(config postgresql.PartitionConfiguration) error { +func (p *PPM) checkPartition(config partition.Configuration) error { p.logger.Debug("Checking partition", "schema", config.Schema, "table", config.Table) err := p.checkPartitionKey(config) @@ -55,40 +65,50 @@ func (p *PPM) checkPartition(config postgresql.PartitionConfiguration) error { return nil } -func (p *PPM) checkPartitionKey(config postgresql.PartitionConfiguration) error { - partition := postgresql.Partition{ - Schema: config.Schema, - ParentTable: config.Table, - Name: config.Table, +func (p *PPM) checkPartitionKey(config partition.Configuration) error { + keyDataType, err := p.db.GetColumnDataType(config.Schema, config.Table, config.PartitionKey) + if err != nil { + return fmt.Errorf("failed to get partition column type: %w", err) } - partitionSettings, err := p.db.GetPartitionSettings(partition.GetParentTable()) + partitionStrategy, partitionKey, err := p.db.GetPartitionSettings(config.Schema, config.Table) if err != nil { return fmt.Errorf("failed to get partition settings: %w", err) } - p.logger.Debug("Partition configuration found", "schema", partition.Schema, "table", partition.Name, "partition_key", partitionSettings.Key, "partition_key_type", partitionSettings.KeyType, "partition_strategy", partitionSettings.Strategy) + p.logger.Debug("Partition configuration found", "schema", config.Schema, "table", config.Table, "partition_key", config.PartitionKey, "partition_key_type", keyDataType, "partition_strategy", partitionStrategy) - if partitionSettings.Key != config.PartitionKey { - p.logger.Warn("Partition key mismatch", "expected", config.PartitionKey, "current", partitionSettings.Key) + if partitionKey != config.PartitionKey { + p.logger.Warn("Partition key mismatch", "expected", config.PartitionKey, "current", partitionKey) return ErrPartitionKeyMismatch } - if !partitionSettings.SupportedStrategy() { + if !IsSupportedStrategy(partitionStrategy) { + p.logger.Warn("Unsupported partition strategy", "strategy", partitionStrategy) + return ErrUnsupportedPartitionStrategy } - if !partitionSettings.SupportedKeyDataType() { + if !IsSupportedKeyDataType(keyDataType) { + p.logger.Warn("Unsupported partition key data type", "partition_key_data_type", keyDataType) + return ErrUnsupportedKeyDataType } return nil } -func (p *PPM) comparePartitions(existingTables, expectedTables []postgresql.Partition) (unexpectedTables, missingTables, incorrectBounds []postgresql.Partition) { - // Maps for tracking presence - existing := make(map[string]postgresql.Partition) +func IsSupportedStrategy(strategy string) bool { + return strategy == string(partition.Range) +} + +func IsSupportedKeyDataType(dataType postgresql.ColumnType) bool { + return slices.Contains(SupportedPartitionKeyDataType, dataType) +} + +func (p *PPM) comparePartitions(existingTables, expectedTables []partition.Partition) (unexpectedTables, missingTables, incorrectBounds []partition.Partition) { + existing := make(map[string]partition.Partition) expectedAndExists := make(map[string]bool) for _, t := range existingTables { @@ -130,17 +150,41 @@ func (p *PPM) comparePartitions(existingTables, expectedTables []postgresql.Part return unexpectedTables, missingTables, incorrectBounds } -func (p *PPM) checkPartitionsConfiguration(partition postgresql.PartitionConfiguration) error { +func (p *PPM) ListPartitions(schema, table string) (partitions []partition.Partition, err error) { + rawPartitions, err := p.db.ListPartitions(schema, table) + if err != nil { + return nil, fmt.Errorf("could not list partitions: %w", err) + } + + for _, p := range rawPartitions { + lowerBound, upperBound, err := parseBounds(p) + if err != nil { + return nil, fmt.Errorf("could not parse bounds: %w", err) + } + + partitions = append(partitions, partition.Partition{ + Schema: p.Schema, + Name: p.Name, + ParentTable: p.ParentTable, + LowerBound: lowerBound, + UpperBound: upperBound, + }) + } + + return partitions, nil +} + +func (p *PPM) checkPartitionsConfiguration(config partition.Configuration) error { partitionContainAnError := false currentTime := time.Now() - expectedPartitions, err := getExpectedPartitions(partition, currentTime) + expectedPartitions, err := getExpectedPartitions(config, currentTime) if err != nil { return fmt.Errorf("could not generate expected partitions: %w", err) } - foundPartitions, err := p.db.ListPartitions(postgresql.Table{Schema: partition.Schema, Name: partition.Table}) + foundPartitions, err := p.ListPartitions(config.Schema, config.Table) if err != nil { return fmt.Errorf("could not list partitions: %w", err) } diff --git a/pkg/ppm/checkpartition_test.go b/pkg/ppm/checkpartition_test.go new file mode 100644 index 0000000..5cab8d3 --- /dev/null +++ b/pkg/ppm/checkpartition_test.go @@ -0,0 +1,200 @@ +package ppm_test + +import ( + "context" + "fmt" + "log/slog" + "testing" + "time" + + "github.com/qonto/postgresql-partition-manager/internal/infra/logger" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/pkg/ppm" + "github.com/qonto/postgresql-partition-manager/pkg/ppm/mocks" + "gotest.tools/assert" +) + +var ( + dayBeforeYesterday = yesterday.AddDate(0, 0, -1) + yesterday = time.Now().AddDate(0, 0, -1) + today = time.Now() + tomorrow = time.Now().AddDate(0, 0, +1) + dayAfterTomorrow = tomorrow.AddDate(0, 0, +1) +) + +func setupMocks(t *testing.T) (*slog.Logger, *mocks.PostgreSQLClient) { + t.Helper() + + logger, err := logger.New(true, "text") + if err != nil { + t.Fatalf("ERROR: Fail to initialize logger: %s", err) + } + + postgreSQLMock := mocks.PostgreSQLClient{} + + return logger, &postgreSQLMock +} + +func TestCheckPartitions(t *testing.T) { + logger, postgreSQLMock := setupMocks(t) + boundDateFormat := "2006-01-02" //nolint:goconst + + partitions := map[string]partition.Configuration{} + partitions["daily partition"] = partition.Configuration{Schema: "app", Table: "daily_table1", PartitionKey: "column", Interval: partition.Daily, Retention: 2, PreProvisioned: 2} + partitions["daily partition without retention"] = partition.Configuration{Schema: "public", Table: "daily_table2", PartitionKey: "created_at", Interval: partition.Daily, Retention: 0, PreProvisioned: 1} + partitions["daily partition without preprovisioned"] = partition.Configuration{Schema: "public", Table: "daily_table3", PartitionKey: "column", Interval: partition.Daily, Retention: 4, PreProvisioned: 0} + partitions["weekly partition"] = partition.Configuration{Schema: "public", Table: "weekly_table", PartitionKey: "weekly", Interval: partition.Weekly, Retention: 2, PreProvisioned: 2} + partitions["monthly partition"] = partition.Configuration{Schema: "public", Table: "monthly_table", PartitionKey: "month", Interval: partition.Monthly, Retention: 2, PreProvisioned: 2} + partitions["yearly partition"] = partition.Configuration{Schema: "public", Table: "yearly_table", PartitionKey: "year", Interval: partition.Yearly, Retention: 4, PreProvisioned: 4} + + // Build mock for each partitions + for _, p := range partitions { + var tables []partition.Partition + + var table partition.Partition + + for i := 0; i <= p.Retention; i++ { + switch p.Interval { + case partition.Daily: + table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i)) + case partition.Weekly: + table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i*7)) + case partition.Monthly: + table, _ = p.GeneratePartition(time.Now().AddDate(0, -i, 0)) + case partition.Yearly: + table, _ = p.GeneratePartition(time.Now().AddDate(-i, 0, 0)) + default: + t.Errorf("unuspported partition interval in retention table mock") + } + + postgreSQLMock.On("GetColumnDataType", table.Schema, table.ParentTable, p.PartitionKey).Return(postgresql.Date, nil).Once() + tables = append(tables, table) + } + + for i := 0; i <= p.PreProvisioned; i++ { + switch p.Interval { + case partition.Daily: + table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i)) + case partition.Weekly: + table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i*7)) + case partition.Monthly: + table, _ = p.GeneratePartition(time.Now().AddDate(0, i, 0)) + case partition.Yearly: + table, _ = p.GeneratePartition(time.Now().AddDate(i, 0, 0)) + default: + t.Errorf("unuspported partition interval in preprovisonned table mock") + } + + postgreSQLMock.On("GetColumnDataType", table.Schema, table.ParentTable, p.PartitionKey).Return(postgresql.Date, nil).Once() + tables = append(tables, table) + } + + postgreSQLMock.On("GetPartitionSettings", p.Schema, p.Table).Return(string(partition.Range), p.PartitionKey, nil).Once() + + convertedTables := partitionResultToPartition(t, tables, boundDateFormat) + postgreSQLMock.On("ListPartitions", p.Schema, p.Table).Return(convertedTables, nil).Once() + } + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, partitions) + assert.NilError(t, checker.CheckPartitions(), "Partitions should succeed") +} + +func TestCheckMissingPartitions(t *testing.T) { + logger, postgreSQLMock := setupMocks(t) + boundDateFormat := "2006-01-02" + + config := partition.Configuration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: partition.Daily, + Retention: 2, + PreProvisioned: 2, + } + + todayPartition, _ := config.GeneratePartition(today) + yesterdayPartition, _ := config.GeneratePartition(yesterday) + tomorrowPartition, _ := config.GeneratePartition(tomorrow) + + testCases := []struct { + name string + tables []partition.Partition + }{ + { + "Missing Yesterday retention partition", + []partition.Partition{ + todayPartition, + yesterdayPartition, + }, + }, + { + "Missing Tomorrow partition", + []partition.Partition{ + todayPartition, + tomorrowPartition, + }, + }, + { + "Missing Today partition", + []partition.Partition{ + yesterdayPartition, + tomorrowPartition, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fmt.Println("tc.tables", tc.tables) + postgreSQLMock.On("GetPartitionSettings", config.Schema, config.Table).Return(string(partition.Range), config.PartitionKey, nil).Once() + postgreSQLMock.On("GetColumnDataType", config.Schema, config.Table, config.PartitionKey).Return(postgresql.Date, nil).Once() + + tables := partitionResultToPartition(t, tc.tables, boundDateFormat) + postgreSQLMock.On("ListPartitions", config.Schema, config.Table).Return(tables, nil).Once() + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, map[string]partition.Configuration{"test": config}) + assert.Error(t, checker.CheckPartitions(), "at least one partition contains an invalid configuration") + }) + } +} + +func TestUnsupportedPartitionsStrategy(t *testing.T) { + logger, postgreSQLMock := setupMocks(t) + + config := partition.Configuration{ + Schema: "public", + Table: "my_table", + PartitionKey: "created_at", + Interval: partition.Daily, + Retention: 2, + PreProvisioned: 2, + } + + testCases := []struct { + name string + strategy partition.PartitionStrategy + key string + }{ + { + "Unsupported list partition strategy", + partition.List, + "created_at", + }, + { + "Unsupported hash partition strategy", + partition.Hash, + "created_at", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + postgreSQLMock.On("GetColumnDataType", config.Schema, config.Table, config.PartitionKey).Return(postgresql.Date, nil).Once() + postgreSQLMock.On("GetPartitionSettings", config.Schema, config.Table).Return(string(tc.strategy), tc.key, nil).Once() + + checker := ppm.New(context.TODO(), *logger, postgreSQLMock, map[string]partition.Configuration{"test": config}) + assert.Error(t, checker.CheckPartitions(), "at least one partition contains an invalid configuration") + }) + } +} diff --git a/pkg/ppm/server.go b/pkg/ppm/checkserver.go similarity index 97% rename from pkg/ppm/server.go rename to pkg/ppm/checkserver.go index 1cc4222..7d66831 100644 --- a/pkg/ppm/server.go +++ b/pkg/ppm/checkserver.go @@ -29,7 +29,7 @@ func (p *PPM) CheckServerRequirements() error { } func (p *PPM) requirePostgreSQLSupportedVersion() error { - version, err := p.db.GetVersion() + version, err := p.db.GetEngineVersion() if err != nil { return fmt.Errorf("failed to fetch PostgreSQL version: %w", err) } diff --git a/pkg/ppm/server_test.go b/pkg/ppm/checkserver_test.go similarity index 96% rename from pkg/ppm/server_test.go rename to pkg/ppm/checkserver_test.go index 1f538bb..8d5a5ce 100644 --- a/pkg/ppm/server_test.go +++ b/pkg/ppm/checkserver_test.go @@ -48,7 +48,7 @@ func TestServerRequirements(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Reset mock on every test case - logger, postgreSQLMock := getTestMocks(t) + logger, postgreSQLMock := setupMocks(t) checker := ppm.New(context.TODO(), *logger, postgreSQLMock, nil) postgreSQLMock.On("GetEngineVersion").Return(tc.serverVersion, nil).Once() diff --git a/pkg/ppm/cleanup.go b/pkg/ppm/cleanup.go index 557eff1..9c7f58d 100644 --- a/pkg/ppm/cleanup.go +++ b/pkg/ppm/cleanup.go @@ -5,10 +5,11 @@ import ( "fmt" "time" - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + partition_pkg "github.com/qonto/postgresql-partition-manager/internal/infra/partition" + "github.com/qonto/postgresql-partition-manager/internal/infra/retry" ) -var ErrPartitionCleanupFailed = errors.New("at least one partition contains could not be cleaned") +var ErrPartitionCleanupFailed = errors.New("at least one partition could not be cleaned") func (p PPM) CleanupPartitions() error { currentTime := time.Now() @@ -22,9 +23,7 @@ func (p PPM) CleanupPartitions() error { return fmt.Errorf("could not generate expected partitions: %w", err) } - parentTable := postgresql.Table{Schema: config.Schema, Name: config.Table} - - foundPartitions, err := p.db.ListPartitions(parentTable) + foundPartitions, err := p.ListPartitions(config.Schema, config.Table) if err != nil { return fmt.Errorf("could not list partitions: %w", err) } @@ -32,7 +31,7 @@ func (p PPM) CleanupPartitions() error { unexpected, _, _ := p.comparePartitions(foundPartitions, expectedPartitions) for _, partition := range unexpected { - err := p.db.DetachPartition(partition) + err := p.DetachPartition(partition) if err != nil { partitionContainAnError = true @@ -41,10 +40,10 @@ func (p PPM) CleanupPartitions() error { continue } - p.logger.Info("Partition detached", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.GetParentTable().Name) + p.logger.Info("Partition detached", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.ParentTable) - if config.CleanupPolicy == postgresql.DropCleanupPolicy { - err := p.db.DeletePartition(partition) + if config.CleanupPolicy == partition_pkg.Drop { + err := p.DeletePartition(partition) if err != nil { partitionContainAnError = true @@ -53,7 +52,7 @@ func (p PPM) CleanupPartitions() error { continue } - p.logger.Info("Partition deleted", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.GetParentTable().Name) + p.logger.Info("Partition deleted", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.ParentTable) } } } @@ -62,5 +61,63 @@ func (p PPM) CleanupPartitions() error { return ErrPartitionCleanupFailed } + p.logger.Info("All partitions are cleaned") + + return nil +} + +func (p PPM) DetachPartition(partition partition_pkg.Partition) error { + p.logger.Debug("Detach partition", "schema", partition.Schema, "table", partition.Name) + + maxRetries := 3 + + err := retry.WithRetry(maxRetries, func(attempt int) error { + err := p.db.DetachPartitionConcurrently(partition.Schema, partition.Name, partition.ParentTable) + if err != nil { + // detachPartitionConcurrently() could fail if the specified partition is in pending detach status + // It could occurred when a previous detach partition concurrently operation was canceled or interrupted + // It prevent any other detach operations on the table + // More info: https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DETACH-PARTITION + // To unblock the situation, we try to finalize the detach operation on Object Not In Prerequisite State error + if isPostgreSQLErrorCode(err, ObjectNotInPrerequisiteStatePostgreSQLErrorCode) { + p.logger.Warn("Table is already pending detach in partitioned, retry with finalize", "error", err, "schema", partition.Schema, "table", partition.Name) + + finalizeErr := p.db.FinalizePartitionDetach(partition.Schema, partition.Name, partition.ParentTable) + if finalizeErr == nil { + err = nil // Returns a success since the partition detach operation has been completed + } + } else { + p.logger.Warn("Fail to detach partition", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) + } + } + + return err + }) + if err != nil { + return fmt.Errorf("failed to detach partition after retries: %w", err) + } + + return nil +} + +func (p PPM) DeletePartition(partition partition_pkg.Partition) error { + p.logger.Debug("Deleting partition", "schema", partition.Schema, "table", partition.Name) + + maxRetries := 3 + + err := retry.WithRetry(maxRetries, func(attempt int) error { + err := p.db.DropTable(partition.Schema, partition.Name) + if err != nil { + p.logger.Warn("Fail to drop table", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) + + return fmt.Errorf("fail to drop table: %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to drop table after retries: %w", err) + } + return nil } diff --git a/pkg/ppm/cleanup_test.go b/pkg/ppm/cleanup_test.go index 87dfef5..5d941aa 100644 --- a/pkg/ppm/cleanup_test.go +++ b/pkg/ppm/cleanup_test.go @@ -5,12 +5,13 @@ import ( "errors" "testing" - "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/jackc/pgx/v5/pgconn" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" "github.com/qonto/postgresql-partition-manager/pkg/ppm" "github.com/stretchr/testify/assert" ) -var OneDayPartitionConfiguration = postgresql.PartitionConfiguration{ +var OneDayPartitionConfiguration = partition.Configuration{ Schema: "public", Table: "my_table", PartitionKey: "created_at", @@ -36,51 +37,52 @@ func TestCleanupPartitions(t *testing.T) { dropPartitionConfiguration := OneDayPartitionConfiguration OneDayPartitionConfiguration.CleanupPolicy = "drop" + boundDateFormat := "2006-01-02" + testCases := []struct { name string - partitions map[string]postgresql.PartitionConfiguration - existingPartitions []postgresql.Partition - expectedRemovedPartitions []postgresql.Partition + partitions map[string]partition.Configuration + existingPartitions []partition.Partition + expectedRemovedPartitions []partition.Partition }{ { "Drop useless partitions", - map[string]postgresql.PartitionConfiguration{ + map[string]partition.Configuration{ "unittest": dropPartitionConfiguration, }, - []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition, yesterdayPartition, currentPartition, tomorrowPartition, dayAfterTomorrowPartition}, - []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition, dayAfterTomorrowPartition}, + []partition.Partition{yearBeforePartition, dayBeforeYesterdayPartition, yesterdayPartition, currentPartition, tomorrowPartition, dayAfterTomorrowPartition}, + []partition.Partition{yearBeforePartition, dayBeforeYesterdayPartition, dayAfterTomorrowPartition}, }, { "Detach useless partitions", - map[string]postgresql.PartitionConfiguration{ + map[string]partition.Configuration{ "unittest": detachPartitionConfiguration, }, - []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition, yesterdayPartition, currentPartition}, - []postgresql.Partition{yearBeforePartition, dayBeforeYesterdayPartition}, + []partition.Partition{yearBeforePartition, dayBeforeYesterdayPartition, yesterdayPartition, currentPartition}, + []partition.Partition{yearBeforePartition, dayBeforeYesterdayPartition}, }, { "No cleanup", - map[string]postgresql.PartitionConfiguration{ + map[string]partition.Configuration{ "unittest": dropPartitionConfiguration, }, - []postgresql.Partition{yesterdayPartition, currentPartition, tomorrowPartition}, - []postgresql.Partition{}, + []partition.Partition{yesterdayPartition, currentPartition, tomorrowPartition}, + []partition.Partition{}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - logger, postgreSQLMock := getTestMocks(t) // Reset mock on every test case + logger, postgreSQLMock := setupMocks(t) // Reset mock on every test case for _, partitionConfiguration := range tc.partitions { - table := postgresql.Table{Schema: partitionConfiguration.Schema, Name: partitionConfiguration.Table} - postgreSQLMock.On("ListPartitions", table).Return(tc.existingPartitions, nil).Once() + postgreSQLMock.On("ListPartitions", partitionConfiguration.Schema, partitionConfiguration.Table).Return(partitionResultToPartition(t, tc.existingPartitions, boundDateFormat), nil).Once() - for _, partition := range tc.expectedRemovedPartitions { - postgreSQLMock.On("DetachPartition", partition).Return(nil).Once() + for _, p := range tc.expectedRemovedPartitions { + postgreSQLMock.On("DetachPartitionConcurrently", p.Schema, p.Name, p.ParentTable).Return(nil).Once() - if partitionConfiguration.CleanupPolicy == postgresql.DropCleanupPolicy { - postgreSQLMock.On("DeletePartition", partition).Return(nil).Once() + if partitionConfiguration.CleanupPolicy == partition.Drop { + postgreSQLMock.On("DropTable", p.Schema, p.Name).Return(nil).Once() } } } @@ -104,45 +106,60 @@ func TestCleanupPartitionsFailover(t *testing.T) { undropablePartitionConfiguration := OneDayPartitionConfiguration undropablePartitionConfiguration.Table = "undropable" - configuration := map[string]postgresql.PartitionConfiguration{ - "undetachable": undetachablePartitionConfiguration, - "undropable": undropablePartitionConfiguration, - "success": successPartitionConfiguration, + pendingFinalizePartitionConfiguration := OneDayPartitionConfiguration + pendingFinalizePartitionConfiguration.Table = "pendingFinalize" + + configuration := map[string]partition.Configuration{ + "undetachable": undetachablePartitionConfiguration, + "undropable": undropablePartitionConfiguration, + "pendingFinalize": pendingFinalizePartitionConfiguration, + "success": successPartitionConfiguration, } - logger, postgreSQLMock := getTestMocks(t) + boundDateFormat := "2006-01-02" - for _, config := range configuration { - table := postgresql.Table{Schema: config.Schema, Name: config.Table} + logger, postgreSQLMock := setupMocks(t) + for _, config := range configuration { dayBeforeYesterdayPartition, _ := config.GeneratePartition(dayBeforeYesterday) yesterdayPartitionPartition, _ := config.GeneratePartition(yesterday) todayPartitionPartition, _ := config.GeneratePartition(today) tomorrowPartitionPartition, _ := config.GeneratePartition(tomorrow) - partitions := []postgresql.Partition{ + partitions := []partition.Partition{ dayBeforeYesterdayPartition, // This partition should be removed by the cleanup yesterdayPartitionPartition, todayPartitionPartition, tomorrowPartitionPartition, } - postgreSQLMock.On("ListPartitions", table).Return(partitions, nil).Once() + postgreSQLMock.On("ListPartitions", config.Schema, config.Table).Return(partitionResultToPartition(t, partitions, boundDateFormat), nil).Once() } // Undetachable partition will return an error on detach operation undetachablePartition, _ := undetachablePartitionConfiguration.GeneratePartition(dayBeforeYesterday) - postgreSQLMock.On("DetachPartition", undetachablePartition).Return(ErrFake).Once() + postgreSQLMock.On("DetachPartitionConcurrently", undetachablePartition.Schema, undetachablePartition.Name, undetachablePartition.ParentTable).Return(ErrFake) // Undropable partition will return an error on drop operation undropablePartition, _ := undropablePartitionConfiguration.GeneratePartition(dayBeforeYesterday) - postgreSQLMock.On("DetachPartition", undropablePartition).Return(nil).Once() - postgreSQLMock.On("DeletePartition", undropablePartition).Return(ErrFake).Once() + postgreSQLMock.On("DetachPartitionConcurrently", undropablePartition.Schema, undropablePartition.Name, undropablePartition.ParentTable).Return(nil) + postgreSQLMock.On("DropTable", undropablePartition.Schema, undropablePartition.Name).Return(ErrFake) + + // Pending finalize partition will return an error on detach operation + pendingFinalizePartition, _ := pendingFinalizePartitionConfiguration.GeneratePartition(dayBeforeYesterday) + ErrObjectNotInPrerequisiteState := &pgconn.PgError{ + Code: ppm.ObjectNotInPrerequisiteStatePostgreSQLErrorCode, + SchemaName: pendingFinalizePartition.Schema, + TableName: pendingFinalizePartition.Name, + } + postgreSQLMock.On("DetachPartitionConcurrently", pendingFinalizePartition.Schema, pendingFinalizePartition.Name, pendingFinalizePartition.ParentTable).Return(ErrObjectNotInPrerequisiteState) + postgreSQLMock.On("FinalizePartitionDetach", pendingFinalizePartition.Schema, pendingFinalizePartition.Name, pendingFinalizePartition.ParentTable).Return(nil) + postgreSQLMock.On("DropTable", pendingFinalizePartition.Schema, pendingFinalizePartition.Name).Return(ErrFake) // Detach and drop will succeed successPartition, _ := successPartitionConfiguration.GeneratePartition(dayBeforeYesterday) - postgreSQLMock.On("DetachPartition", successPartition).Return(nil).Once() - postgreSQLMock.On("DeletePartition", successPartition).Return(nil).Once() + postgreSQLMock.On("DetachPartitionConcurrently", successPartition.Schema, successPartition.Name, successPartition.ParentTable).Return(nil).Once() + postgreSQLMock.On("DropTable", successPartition.Schema, successPartition.Name).Return(nil).Once() checker := ppm.New(context.TODO(), *logger, postgreSQLMock, configuration) err := checker.CleanupPartitions() diff --git a/pkg/ppm/error.go b/pkg/ppm/error.go new file mode 100644 index 0000000..31f22da --- /dev/null +++ b/pkg/ppm/error.go @@ -0,0 +1,17 @@ +package ppm + +import ( + "errors" + + "github.com/jackc/pgx/v5/pgconn" +) + +const ( + ObjectNotInPrerequisiteStatePostgreSQLErrorCode = "55000" +) + +func isPostgreSQLErrorCode(err error, errorCode string) bool { + var pgErr *pgconn.PgError + + return errors.As(err, &pgErr) && pgErr.Code == errorCode +} diff --git a/pkg/ppm/error_internal_test.go b/pkg/ppm/error_internal_test.go new file mode 100644 index 0000000..5fee8ec --- /dev/null +++ b/pkg/ppm/error_internal_test.go @@ -0,0 +1,45 @@ +package ppm + +import ( + "errors" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "gotest.tools/assert" +) + +var ErrGeneric = errors.New("a generic error") + +func TestPostgreSQLError(t *testing.T) { + testCases := []struct { + name string + error error + code string + expected bool + }{ + { + name: "ObjectNotInPrerequisiteState", + error: &pgconn.PgError{Code: "55000"}, + code: "55000", + expected: true, + }, + { + name: "Non match error code", + error: &pgconn.PgError{Code: "42"}, + code: "55000", + expected: false, + }, + { + name: "Generic error", + error: ErrGeneric, + code: "", + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, isPostgreSQLErrorCode(tc.error, tc.code), tc.expected, "Error code should match") + }) + } +} diff --git a/pkg/ppm/mocks/PostgreSQLClient.go b/pkg/ppm/mocks/PostgreSQLClient.go index 99293fc..329d4fd 100644 --- a/pkg/ppm/mocks/PostgreSQLClient.go +++ b/pkg/ppm/mocks/PostgreSQLClient.go @@ -14,13 +14,13 @@ type PostgreSQLClient struct { mock.Mock } -// CreatePartition provides a mock function with given fields: partitionConfiguration, partition -func (_m *PostgreSQLClient) CreatePartition(partitionConfiguration postgresql.PartitionConfiguration, partition postgresql.Partition) error { - ret := _m.Called(partitionConfiguration, partition) +// AttachPartition provides a mock function with given fields: schema, table, parent, lowerBound, upperBound +func (_m *PostgreSQLClient) AttachPartition(schema string, table string, parent string, lowerBound string, upperBound string) error { + ret := _m.Called(schema, table, parent, lowerBound, upperBound) var r0 error - if rf, ok := ret.Get(0).(func(postgresql.PartitionConfiguration, postgresql.Partition) error); ok { - r0 = rf(partitionConfiguration, partition) + if rf, ok := ret.Get(0).(func(string, string, string, string, string) error); ok { + r0 = rf(schema, table, parent, lowerBound, upperBound) } else { r0 = ret.Error(0) } @@ -28,13 +28,13 @@ func (_m *PostgreSQLClient) CreatePartition(partitionConfiguration postgresql.Pa return r0 } -// DeletePartition provides a mock function with given fields: partition -func (_m *PostgreSQLClient) DeletePartition(partition postgresql.Partition) error { - ret := _m.Called(partition) +// CreateTableLikeTable provides a mock function with given fields: schema, table, parent +func (_m *PostgreSQLClient) CreateTableLikeTable(schema string, table string, parent string) error { + ret := _m.Called(schema, table, parent) var r0 error - if rf, ok := ret.Get(0).(func(postgresql.Partition) error); ok { - r0 = rf(partition) + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(schema, table, parent) } else { r0 = ret.Error(0) } @@ -42,13 +42,13 @@ func (_m *PostgreSQLClient) DeletePartition(partition postgresql.Partition) erro return r0 } -// DetachPartition provides a mock function with given fields: partition -func (_m *PostgreSQLClient) DetachPartition(partition postgresql.Partition) error { - ret := _m.Called(partition) +// DetachPartitionConcurrently provides a mock function with given fields: schema, table, parent +func (_m *PostgreSQLClient) DetachPartitionConcurrently(schema string, table string, parent string) error { + ret := _m.Called(schema, table, parent) var r0 error - if rf, ok := ret.Get(0).(func(postgresql.Partition) error); ok { - r0 = rf(partition) + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(schema, table, parent) } else { r0 = ret.Error(0) } @@ -56,23 +56,51 @@ func (_m *PostgreSQLClient) DetachPartition(partition postgresql.Partition) erro return r0 } -// GetPartitionSettings provides a mock function with given fields: _a0 -func (_m *PostgreSQLClient) GetPartitionSettings(_a0 postgresql.Table) (postgresql.PartitionSettings, error) { - ret := _m.Called(_a0) +// DropTable provides a mock function with given fields: schema, table +func (_m *PostgreSQLClient) DropTable(schema string, table string) error { + ret := _m.Called(schema, table) - var r0 postgresql.PartitionSettings + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(schema, table) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// FinalizePartitionDetach provides a mock function with given fields: schema, table, parent +func (_m *PostgreSQLClient) FinalizePartitionDetach(schema string, table string, parent string) error { + ret := _m.Called(schema, table, parent) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(schema, table, parent) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetColumnDataType provides a mock function with given fields: schema, table, column +func (_m *PostgreSQLClient) GetColumnDataType(schema string, table string, column string) (postgresql.ColumnType, error) { + ret := _m.Called(schema, table, column) + + var r0 postgresql.ColumnType var r1 error - if rf, ok := ret.Get(0).(func(postgresql.Table) (postgresql.PartitionSettings, error)); ok { - return rf(_a0) + if rf, ok := ret.Get(0).(func(string, string, string) (postgresql.ColumnType, error)); ok { + return rf(schema, table, column) } - if rf, ok := ret.Get(0).(func(postgresql.Table) postgresql.PartitionSettings); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(string, string, string) postgresql.ColumnType); ok { + r0 = rf(schema, table, column) } else { - r0 = ret.Get(0).(postgresql.PartitionSettings) + r0 = ret.Get(0).(postgresql.ColumnType) } - if rf, ok := ret.Get(1).(func(postgresql.Table) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(schema, table, column) } else { r1 = ret.Error(1) } @@ -80,6 +108,61 @@ func (_m *PostgreSQLClient) GetPartitionSettings(_a0 postgresql.Table) (postgres return r0, r1 } +// GetEngineVersion provides a mock function with given fields: +func (_m *PostgreSQLClient) GetEngineVersion() (int64, error) { + ret := _m.Called() + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func() (int64, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetPartitionSettings provides a mock function with given fields: schema, table +func (_m *PostgreSQLClient) GetPartitionSettings(schema string, table string) (string, string, error) { + ret := _m.Called(schema, table) + + var r0 string + var r1 string + var r2 error + if rf, ok := ret.Get(0).(func(string, string) (string, string, error)); ok { + return rf(schema, table) + } + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(schema, table) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string) string); ok { + r1 = rf(schema, table) + } else { + r1 = ret.Get(1).(string) + } + + if rf, ok := ret.Get(2).(func(string, string) error); ok { + r2 = rf(schema, table) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + // GetServerTime provides a mock function with given fields: func (_m *PostgreSQLClient) GetServerTime() (time.Time, error) { ret := _m.Called() @@ -104,23 +187,47 @@ func (_m *PostgreSQLClient) GetServerTime() (time.Time, error) { return r0, r1 } -// GetVersion provides a mock function with given fields: -func (_m *PostgreSQLClient) GetVersion() (int64, error) { - ret := _m.Called() +// IsPartitionAttached provides a mock function with given fields: schema, table +func (_m *PostgreSQLClient) IsPartitionAttached(schema string, table string) (bool, error) { + ret := _m.Called(schema, table) - var r0 int64 + var r0 bool var r1 error - if rf, ok := ret.Get(0).(func() (int64, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(string, string) (bool, error)); ok { + return rf(schema, table) } - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(schema, table) } else { - r0 = ret.Get(0).(int64) + r0 = ret.Get(0).(bool) } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(schema, table) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IsTableExists provides a mock function with given fields: schema, table +func (_m *PostgreSQLClient) IsTableExists(schema string, table string) (bool, error) { + ret := _m.Called(schema, table) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (bool, error)); ok { + return rf(schema, table) + } + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(schema, table) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(schema, table) } else { r1 = ret.Error(1) } @@ -128,25 +235,25 @@ func (_m *PostgreSQLClient) GetVersion() (int64, error) { return r0, r1 } -// ListPartitions provides a mock function with given fields: table -func (_m *PostgreSQLClient) ListPartitions(table postgresql.Table) ([]postgresql.Partition, error) { - ret := _m.Called(table) +// ListPartitions provides a mock function with given fields: schema, table +func (_m *PostgreSQLClient) ListPartitions(schema string, table string) ([]postgresql.PartitionResult, error) { + ret := _m.Called(schema, table) - var r0 []postgresql.Partition + var r0 []postgresql.PartitionResult var r1 error - if rf, ok := ret.Get(0).(func(postgresql.Table) ([]postgresql.Partition, error)); ok { - return rf(table) + if rf, ok := ret.Get(0).(func(string, string) ([]postgresql.PartitionResult, error)); ok { + return rf(schema, table) } - if rf, ok := ret.Get(0).(func(postgresql.Table) []postgresql.Partition); ok { - r0 = rf(table) + if rf, ok := ret.Get(0).(func(string, string) []postgresql.PartitionResult); ok { + r0 = rf(schema, table) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]postgresql.Partition) + r0 = ret.Get(0).([]postgresql.PartitionResult) } } - if rf, ok := ret.Get(1).(func(postgresql.Table) error); ok { - r1 = rf(table) + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(schema, table) } else { r1 = ret.Error(1) } @@ -159,8 +266,7 @@ func (_m *PostgreSQLClient) ListPartitions(table postgresql.Table) ([]postgresql func NewPostgreSQLClient(t interface { mock.TestingT Cleanup(func()) -}, -) *PostgreSQLClient { +}) *PostgreSQLClient { mock := &PostgreSQLClient{} mock.Mock.Test(t) diff --git a/pkg/ppm/ppm.go b/pkg/ppm/ppm.go index 8800ea4..9235c62 100644 --- a/pkg/ppm/ppm.go +++ b/pkg/ppm/ppm.go @@ -7,27 +7,33 @@ import ( "log/slog" "time" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" ) type PostgreSQLClient interface { - CreatePartition(partitionConfiguration postgresql.PartitionConfiguration, partition postgresql.Partition) error - DeletePartition(partition postgresql.Partition) error - DetachPartition(partition postgresql.Partition) error - GetPartitionSettings(postgresql.Table) (postgresql.PartitionSettings, error) - ListPartitions(table postgresql.Table) ([]postgresql.Partition, error) - GetVersion() (int64, error) + ListPartitions(schema, table string) (partitions []postgresql.PartitionResult, err error) + GetEngineVersion() (int64, error) GetServerTime() (time.Time, error) + IsTableExists(schema, table string) (bool, error) + IsPartitionAttached(schema, table string) (bool, error) + AttachPartition(schema, table, parent, lowerBound, upperBound string) error + CreateTableLikeTable(schema, table, parent string) error + GetColumnDataType(schema, table, column string) (postgresql.ColumnType, error) + GetPartitionSettings(schema, table string) (strategy, key string, err error) + DropTable(schema, table string) error + DetachPartitionConcurrently(schema, table, parent string) error + FinalizePartitionDetach(schema, table, parent string) error } type PPM struct { ctx context.Context db PostgreSQLClient - partitions map[string]postgresql.PartitionConfiguration + partitions map[string]partition.Configuration logger slog.Logger } -func New(context context.Context, logger slog.Logger, db PostgreSQLClient, partitions map[string]postgresql.PartitionConfiguration) *PPM { +func New(context context.Context, logger slog.Logger, db PostgreSQLClient, partitions map[string]partition.Configuration) *PPM { return &PPM{ partitions: partitions, ctx: context, @@ -36,7 +42,7 @@ func New(context context.Context, logger slog.Logger, db PostgreSQLClient, parti } } -func getExpectedPartitions(partition postgresql.PartitionConfiguration, currentTime time.Time) (partitions []postgresql.Partition, err error) { +func getExpectedPartitions(partition partition.Configuration, currentTime time.Time) (partitions []partition.Partition, err error) { retentions, err := partition.GetRetentionPartitions(currentTime) if err != nil { return partitions, fmt.Errorf("could not generate retention partitions: %w", err) diff --git a/pkg/ppm/ppm_test.go b/pkg/ppm/ppm_test.go new file mode 100644 index 0000000..95e9389 --- /dev/null +++ b/pkg/ppm/ppm_test.go @@ -0,0 +1,58 @@ +package ppm_test + +import ( + "testing" + + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" + "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" +) + +func formatLowerBound(t *testing.T, p partition.Partition, config partition.Configuration) (output string) { + t.Helper() + + var dateFormat string + + switch config.Interval { + case partition.Daily, partition.Weekly: + dateFormat = "2006-01-02" + case partition.Monthly: + dateFormat = "2006-01" + case partition.Yearly: + dateFormat = "2006" + } + + return p.LowerBound.Format(dateFormat) +} + +func formatUpperBound(t *testing.T, p partition.Partition, config partition.Configuration) (output string) { + t.Helper() + + var dateFormat string + + switch config.Interval { + case partition.Daily, partition.Weekly: + dateFormat = "2006-01-02" + case partition.Monthly: + dateFormat = "2006-01" + case partition.Yearly: + dateFormat = "2006" + } + + return p.UpperBound.Format(dateFormat) +} + +func partitionResultToPartition(t *testing.T, partitions []partition.Partition, dateFormat string) (result []postgresql.PartitionResult) { + t.Helper() + + for _, p := range partitions { + result = append(result, postgresql.PartitionResult{ + ParentTable: p.ParentTable, + Schema: p.Schema, + Name: p.Name, + LowerBound: p.LowerBound.Format(dateFormat), + UpperBound: p.UpperBound.Format(dateFormat), + }) + } + + return +} diff --git a/pkg/ppm/provisioning.go b/pkg/ppm/provisioning.go index 2ee1dad..d90a72a 100644 --- a/pkg/ppm/provisioning.go +++ b/pkg/ppm/provisioning.go @@ -5,7 +5,10 @@ import ( "fmt" "time" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" + "github.com/qonto/postgresql-partition-manager/internal/infra/retry" + "github.com/qonto/postgresql-partition-manager/internal/infra/uuid7" ) var ErrPartitionProvisioningFailed = errors.New("partition provisioning failed for one or more partition") @@ -26,10 +29,12 @@ func (p PPM) ProvisioningPartitions() error { return ErrPartitionProvisioningFailed } + p.logger.Info("All partitions are correctly provisioned") + return nil } -func (p PPM) provisionPartitionsFor(config postgresql.PartitionConfiguration, at time.Time) error { +func (p PPM) provisionPartitionsFor(config partition.Configuration, at time.Time) error { provisioningFailed := false partitions, err := getExpectedPartitions(config, at) @@ -38,7 +43,7 @@ func (p PPM) provisionPartitionsFor(config postgresql.PartitionConfiguration, at } for _, partition := range partitions { - if err := p.db.CreatePartition(config, partition); err != nil { + if err := p.CreatePartition(config, partition); err != nil { provisioningFailed = true p.logger.Error("Failed to create partition", "error", err, "schema", partition.Schema, "table", partition.Name) @@ -51,3 +56,80 @@ func (p PPM) provisionPartitionsFor(config postgresql.PartitionConfiguration, at return nil } + +func (p PPM) CreatePartition(partitionConfiguration partition.Configuration, partition partition.Partition) error { + p.logger.Debug("Creating partition", "schema", partition.Schema, "table", partition.Name) + + tableExists, err := p.db.IsTableExists(partition.Schema, partition.Name) + if err != nil { + return fmt.Errorf("failed to check if table exists: %w", err) + } + + if !tableExists { + err := p.db.CreateTableLikeTable(partition.Schema, partition.Name, partition.ParentTable) + if err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + + p.logger.Info("Table created", "schema", partition.Schema, "table", partition.Name) + } else { + p.logger.Info("Table already exists, skip", "schema", partition.Schema, "table", partition.Name) + } + + partitionAttached, err := p.db.IsPartitionAttached(partition.Schema, partition.Name) + if err != nil { + return fmt.Errorf("failed to check partition attachment status: %w", err) + } + + if partitionAttached { + p.logger.Info("Table is already attached to the parent table, skip", "schema", partition.Schema, "table", partition.Name) + + return nil + } + + _, partitionKey, err := p.db.GetPartitionSettings(partition.Schema, partition.ParentTable) + if err != nil { + return fmt.Errorf("failed to get partition settings: %w", err) + } + + partitionKeyType, err := p.db.GetColumnDataType(partition.Schema, partition.ParentTable, partitionKey) + if err != nil { + return fmt.Errorf("failed to get partition settings: %w", err) + } + + var lowerBound, upperBound string + + switch partitionKeyType { + case postgresql.Date: + lowerBound = partition.LowerBound.Format("2006-01-02") + upperBound = partition.UpperBound.Format("2006-01-02") + case postgresql.DateTime: + lowerBound = partition.LowerBound.Format("2006-01-02 00:00:00") + upperBound = partition.UpperBound.Format("2006-01-02 00:00:00") + case postgresql.UUID: + lowerBound = uuid7.FromTime(partition.LowerBound) + upperBound = uuid7.FromTime(partition.UpperBound) + default: + return ErrUnsupportedPartitionStrategy + } + + maxRetries := 3 + + err = retry.WithRetry(maxRetries, func(attempt int) error { + err := p.db.AttachPartition(partition.Schema, partition.Name, partition.ParentTable, lowerBound, upperBound) + if err != nil { + p.logger.Warn("fail to attach partition", "error", err, "schema", partition.Schema, "table", partition.Name, "attempt", attempt, "max_retries", maxRetries) + + return fmt.Errorf("fail to attach partition: %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to attach partition after retries: %w", err) + } + + p.logger.Info("Partition attached to parent table", "schema", partition.Schema, "table", partition.Name, "parent_table", partition.ParentTable) + + return nil +} diff --git a/pkg/ppm/provisioning_test.go b/pkg/ppm/provisioning_test.go index fd391fb..02aae22 100644 --- a/pkg/ppm/provisioning_test.go +++ b/pkg/ppm/provisioning_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/qonto/postgresql-partition-manager/internal/infra/partition" "github.com/qonto/postgresql-partition-manager/internal/infra/postgresql" "github.com/qonto/postgresql-partition-manager/pkg/ppm" "github.com/stretchr/testify/assert" @@ -22,15 +23,15 @@ func TestProvisioning(t *testing.T) { testCases := []struct { name string - partitions map[string]postgresql.PartitionConfiguration - expectedCreatedPartitions []postgresql.Partition + partitions map[string]partition.Configuration + expectedCreatedPartitions []partition.Partition }{ { "Provisioning create preProvisioned and retention partitions", - map[string]postgresql.PartitionConfiguration{ + map[string]partition.Configuration{ "unittest": OneDayPartitionConfiguration, }, - []postgresql.Partition{ + []partition.Partition{ yesterdayPartition, currentPartition, tomorrowPartition, @@ -38,11 +39,11 @@ func TestProvisioning(t *testing.T) { }, { "Multiple provisioning", - map[string]postgresql.PartitionConfiguration{ + map[string]partition.Configuration{ "unittest 1": TwoDayPartitionConfiguration, "unittest 2": TwoDayPartitionConfiguration, }, - []postgresql.Partition{ + []partition.Partition{ dayBeforeYesterdayPartition, yesterdayPartition, currentPartition, @@ -54,11 +55,16 @@ func TestProvisioning(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - logger, postgreSQLMock := getTestMocks(t) // Reset mock on every test case - - for _, partitionConfiguration := range tc.partitions { - for _, partition := range tc.expectedCreatedPartitions { - postgreSQLMock.On("CreatePartition", partitionConfiguration, partition).Return(nil).Once() + logger, postgreSQLMock := setupMocks(t) // Reset mock on every test case + + for _, configuration := range tc.partitions { + for _, p := range tc.expectedCreatedPartitions { + postgreSQLMock.On("IsTableExists", p.Schema, p.Name).Return(false, nil).Once() + postgreSQLMock.On("IsPartitionAttached", p.Schema, p.Name).Return(false, nil).Once() + postgreSQLMock.On("GetPartitionSettings", p.Schema, p.ParentTable).Return(string(partition.Range), configuration.PartitionKey, nil).Once() + postgreSQLMock.On("GetColumnDataType", p.Schema, p.ParentTable, configuration.PartitionKey).Return(postgresql.Date, nil).Once() + postgreSQLMock.On("CreateTableLikeTable", p.Schema, p.Name, p.ParentTable).Return(nil).Once() + postgreSQLMock.On("AttachPartition", p.Schema, p.Name, p.ParentTable, formatLowerBound(t, p, configuration), formatUpperBound(t, p, configuration)).Return(nil).Once() } } @@ -73,12 +79,14 @@ func TestProvisioning(t *testing.T) { // Test provisioning continue even if a partition could not be created func TestProvisioningFailover(t *testing.T) { + successPartitionConfiguration := OneDayPartitionConfiguration + failedPartitionConfiguration := OneDayPartitionConfiguration failedPartitionConfiguration.Table = "failed" testCases := []struct { name string - config postgresql.PartitionConfiguration + config partition.Configuration createPartitionError error }{ { @@ -88,26 +96,34 @@ func TestProvisioningFailover(t *testing.T) { }, { "success provisioning", - OneDayPartitionConfiguration, + successPartitionConfiguration, nil, }, } - logger, postgreSQLMock := getTestMocks(t) + logger, postgreSQLMock := setupMocks(t) - configuration := map[string]postgresql.PartitionConfiguration{} + configuration := map[string]partition.Configuration{} for _, tc := range testCases { previous, _ := tc.config.GetRetentionPartitions(today) current, _ := tc.config.GeneratePartition(today) future, _ := tc.config.GetPreProvisionedPartitions(today) - partitions := []postgresql.Partition{current} + partitions := []partition.Partition{current} partitions = append(partitions, previous...) partitions = append(partitions, future...) - for _, partition := range partitions { - postgreSQLMock.On("CreatePartition", tc.config, partition).Return(tc.createPartitionError).Once() + for _, p := range partitions { + postgreSQLMock.On("IsTableExists", p.Schema, p.Name).Return(false, nil).Once() + postgreSQLMock.On("CreateTableLikeTable", p.Schema, p.Name, p.ParentTable).Return(tc.createPartitionError).Once() + + if tc.createPartitionError == nil { + postgreSQLMock.On("IsPartitionAttached", p.Schema, p.Name).Return(false, nil) + postgreSQLMock.On("GetPartitionSettings", p.Schema, p.ParentTable).Return(string(partition.Range), tc.config.PartitionKey, nil).Once() + postgreSQLMock.On("GetColumnDataType", p.Schema, p.ParentTable, tc.config.PartitionKey).Return(postgresql.Date, nil).Once() + postgreSQLMock.On("AttachPartition", p.Schema, p.Name, p.ParentTable, formatLowerBound(t, p, tc.config), formatUpperBound(t, p, tc.config)).Return(nil).Once() + } } configuration[tc.name] = tc.config From 6d86fcc7bf04592e6020bf88e04265ab4e5570f4 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Wed, 24 Apr 2024 11:11:42 +0200 Subject: [PATCH 23/27] refact(cmd): Rewrite package to split PostgreSQL logic --- cmd/run/run.go | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/cmd/run/run.go b/cmd/run/run.go index 2eef93d..ae9c2c4 100644 --- a/cmd/run/run.go +++ b/cmd/run/run.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "log/slog" "os" "github.com/qonto/postgresql-partition-manager/internal/infra/config" @@ -50,11 +49,11 @@ var AllCmd = &cobra.Command{ Short: "Perform partitions provisioning, cleanup, and check", Long: "Perform partitions provisioning, cleanup, and check", Run: func(cmd *cobra.Command, args []string) { - client, logger := initCmd() + client := initCmd() - provisioningCmd(client, logger) - cleanupCmd(client, logger) - checkCmd(client, logger) + provisioningCmd(client) + cleanupCmd(client) + checkCmd(client) }, } @@ -63,8 +62,8 @@ var CheckCmd = &cobra.Command{ Short: "Check existing partitions", Long: "Check existing partitions", Run: func(cmd *cobra.Command, args []string) { - client, logger := initCmd() - checkCmd(client, logger) + client := initCmd() + checkCmd(client) }, } @@ -73,8 +72,8 @@ var CleanupCmd = &cobra.Command{ Short: "Remove outdated partitions", Long: "Remove outdated partitions", Run: func(cmd *cobra.Command, args []string) { - client, logger := initCmd() - cleanupCmd(client, logger) + client := initCmd() + cleanupCmd(client) }, } @@ -83,12 +82,12 @@ var ProvisioningCmd = &cobra.Command{ Short: "Create and attach new partitions", Long: "Create and attach new partitions", Run: func(cmd *cobra.Command, args []string) { - client, logger := initCmd() - provisioningCmd(client, logger) + client := initCmd() + provisioningCmd(client) }, } -func initCmd() (*ppm.PPM, *slog.Logger) { +func initCmd() *ppm.PPM { var config config.Config if err := viper.Unmarshal(&config); err != nil { @@ -128,29 +127,23 @@ func initCmd() (*ppm.PPM, *slog.Logger) { os.Exit(DatabaseErrorExitCode) } - return client, log + return client } -func checkCmd(client *ppm.PPM, logger *slog.Logger) { +func checkCmd(client *ppm.PPM) { if err := client.CheckPartitions(); err != nil { os.Exit(PartitionsCheckFailedExitCode) } - - logger.Info("All partitions are correctly configured") } -func cleanupCmd(client *ppm.PPM, logger *slog.Logger) { +func cleanupCmd(client *ppm.PPM) { if err := client.CleanupPartitions(); err != nil { os.Exit(PartitionsCleanupFailedExitCode) } - - logger.Info("All partitions are cleaned") } -func provisioningCmd(client *ppm.PPM, logger *slog.Logger) { +func provisioningCmd(client *ppm.PPM) { if err := client.ProvisioningPartitions(); err != nil { os.Exit(PartitionsProvisioningFailedExitCode) } - - logger.Info("All partitions are correctly provisioned") } From b67073347965790ef3c8e55ee1af3a82a5c73dba Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 12:24:51 +0200 Subject: [PATCH 24/27] chore(kubernetesdev): Add local development environment --- .pre-commit-config.yaml | 2 +- .yamllint.yaml | 2 + CONTRIBUTING.md | 10 +- Dockerfile | 31 +++++ README.md | 2 +- scripts/kubernetesdev/.gitignore | 2 + scripts/kubernetesdev/Chart.yaml | 13 +++ .../templates/configmap_configuration.yaml | 9 ++ .../templates/configmap_seeds.yaml | 16 +++ .../kubernetesdev/templates/deployment.yaml | 110 ++++++++++++++++++ scripts/kubernetesdev/templates/pv.yaml | 17 +++ scripts/kubernetesdev/templates/pvc.yaml | 14 +++ scripts/kubernetesdev/templates/secret.yaml | 9 ++ scripts/kubernetesdev/templates/service.yaml | 28 +++++ scripts/kubernetesdev/values.yaml | 33 ++++++ 15 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 Dockerfile create mode 100644 scripts/kubernetesdev/.gitignore create mode 100644 scripts/kubernetesdev/Chart.yaml create mode 100644 scripts/kubernetesdev/templates/configmap_configuration.yaml create mode 100644 scripts/kubernetesdev/templates/configmap_seeds.yaml create mode 100644 scripts/kubernetesdev/templates/deployment.yaml create mode 100644 scripts/kubernetesdev/templates/pv.yaml create mode 100644 scripts/kubernetesdev/templates/pvc.yaml create mode 100644 scripts/kubernetesdev/templates/secret.yaml create mode 100644 scripts/kubernetesdev/templates/service.yaml create mode 100644 scripts/kubernetesdev/values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d3b91d..03f120c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: hooks: - id: trailing-whitespace - id: check-yaml - exclude: configs/helm/templates/ + exclude: configs/helm/templates/|scripts/kubernetesdev/templates/ - id: check-json - id: end-of-file-fixer - id: detect-private-key diff --git a/.yamllint.yaml b/.yamllint.yaml index dabaed0..727062a 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -9,3 +9,5 @@ rules: ignore: - dist/ - configs/helm/templates/*.yaml # Helm templates are invalid due to Go template language + - scripts/kubernetesdev/templates/configmap_configuration.yaml + - scripts/kubernetesdev/templates/secret.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8aa025..6762d9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,9 +179,13 @@ Steps: HELM_RELEASE_NAME=main # Replace with an helm release ``` -1. Trigger PostgreSQL and Postgresql Partition Manager deployments +1. Optional. Adjust deployment settings in `values.yaml`. - Optional. Adjust deployment settings in `values.yaml`. + ```bash + vim values.yaml + ``` + +1. Trigger PostgreSQL and Postgresql Partition Manager deployments ```bash helm upgrade ${HELM_RELEASE_NAME} . --install --values values.yaml @@ -235,7 +239,7 @@ Connect to PostgreSQL ```bash export PGHOST=localhost -export PGPORT=$(kubectl get svc postgres -o jsonpath='{.spec.ports[0].nodePort}') +export PGPORT=$(kubectl get svc postgres-nodeport -o jsonpath='{.spec.ports[0].nodePort}') export PGUSER=$(kubectl get secret postgres-credentials --template={{.data.user}} | base64 -D) export PGPASSWORD=$(kubectl get secret postgres-credentials --template={{.data.password}} | base64 -D) export PGDATABASE=$(kubectl get configmap postgres-configuration --template={{.data.database}}) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..441801a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.22 AS builder + +WORKDIR /build + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . . + +HEALTHCHECK NONE + +RUN make build + + +FROM alpine:3.19 + +ARG USER=app +ARG HOME=/app + +RUN addgroup -g 1001 -S app \ + && adduser --home /app -u 1001 -S app -G app \ + && mkdir -p /app \ + && chown app:app -R /app + +WORKDIR $HOME +USER $USER + +COPY --from=builder /build/postgresql-partition-manager $HOME/postgresql-partition-manager + +ENTRYPOINT [ "/app/postgresql-partition-manager" ] diff --git a/README.md b/README.md index 7e8988f..a2edfcf 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ PPM is available as a Docker image, Debian package, and Binary. --version ${POSTGRESQL_PARTION_MANAGER} \ --install \ --namespace ${KUBERNETES_NAMESPACE} - --values + --values values.yaml ``` 1. Trigger job manually and verify application logs diff --git a/scripts/kubernetesdev/.gitignore b/scripts/kubernetesdev/.gitignore new file mode 100644 index 0000000..d7be5da --- /dev/null +++ b/scripts/kubernetesdev/.gitignore @@ -0,0 +1,2 @@ +/charts/ +/Chart.lock diff --git a/scripts/kubernetesdev/Chart.yaml b/scripts/kubernetesdev/Chart.yaml new file mode 100644 index 0000000..2e3de98 --- /dev/null +++ b/scripts/kubernetesdev/Chart.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v2 +type: application +name: postgresql-partition-manager +description: Manages PostgreSQL partition + +version: 0.0.0 +appVersion: 0.0.0 + +dependencies: + - name: postgresql-partition-manager + version: "~0" + repository: "file://../../configs/helm" diff --git a/scripts/kubernetesdev/templates/configmap_configuration.yaml b/scripts/kubernetesdev/templates/configmap_configuration.yaml new file mode 100644 index 0000000..af294ae --- /dev/null +++ b/scripts/kubernetesdev/templates/configmap_configuration.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-configuration + labels: + app: postgres +data: + database: {{.Values.postgresDatabase}} diff --git a/scripts/kubernetesdev/templates/configmap_seeds.yaml b/scripts/kubernetesdev/templates/configmap_seeds.yaml new file mode 100644 index 0000000..f8b827b --- /dev/null +++ b/scripts/kubernetesdev/templates/configmap_seeds.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-seeds + labels: + app: postgres +data: + seeds: | + \c {{ .Values.postgresDatabase }}; + + CREATE TABLE by_date ( + id BIGSERIAL, + temperature INT, + created_at DATE NOT NULL + ) PARTITION BY RANGE (created_at); diff --git a/scripts/kubernetesdev/templates/deployment.yaml b/scripts/kubernetesdev/templates/deployment.yaml new file mode 100644 index 0000000..c6e79cd --- /dev/null +++ b/scripts/kubernetesdev/templates/deployment.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: 'postgres:16' + imagePullPolicy: Always + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-configuration + key: database + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: password + volumeMounts: + - name: postgresdata + mountPath: /var/lib/postgresql/data + - name: seeds + mountPath: /docker-entrypoint-initdb.d/seeds.sql + subPath: seeds.sql + readOnly: true + readinessProbe: + exec: + command: + - pg_isready + - "-U" + - "{{ .Values.postgresUser }}" + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + livenessProbe: + exec: + command: + - pg_isready + - "-U" + - "{{ .Values.postgresUser }}" + initialDelaySeconds: 15 + timeoutSeconds: 5 + periodSeconds: 20 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + exec: + command: + - pg_isready + - "-U" + - "{{ .Values.postgresUser }}" + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + args: + - "-c" + - log_statement=all + - "-c" + - "log_line_prefix='%t:%r:user=%u,database=%d,app=%a,query_id=%Q:[%p]:'" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 128Mi + volumes: + - name: postgresdata + persistentVolumeClaim: + claimName: postgres-volume-claim + - name: seeds + configMap: + name: postgres-seeds + items: + - key: seeds + path: seeds.sql diff --git a/scripts/kubernetesdev/templates/pv.yaml b/scripts/kubernetesdev/templates/pv.yaml new file mode 100644 index 0000000..6b9c43c --- /dev/null +++ b/scripts/kubernetesdev/templates/pv.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: postgres-volume + labels: + type: local + app: postgres +spec: + storageClassName: manual + reclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + hostPath: + path: /data/postgresql/2024-04-25-008 diff --git a/scripts/kubernetesdev/templates/pvc.yaml b/scripts/kubernetesdev/templates/pvc.yaml new file mode 100644 index 0000000..68a203f --- /dev/null +++ b/scripts/kubernetesdev/templates/pvc.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-volume-claim + labels: + app: postgres +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/scripts/kubernetesdev/templates/secret.yaml b/scripts/kubernetesdev/templates/secret.yaml new file mode 100644 index 0000000..eff72e3 --- /dev/null +++ b/scripts/kubernetesdev/templates/secret.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials +type: Opaque +data: + user: {{.Values.postgresUser | b64enc}} # yamllint disable-line + password: {{.Values.postgresPassword | b64enc}} # yamllint disable-line diff --git a/scripts/kubernetesdev/templates/service.yaml b/scripts/kubernetesdev/templates/service.yaml new file mode 100644 index 0000000..7411682 --- /dev/null +++ b/scripts/kubernetesdev/templates/service.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-nodeport + labels: + app: postgres +spec: + type: NodePort + ports: + - protocol: TCP + port: 5432 + selector: + app: postgres +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 5432 + selector: + app: postgres diff --git a/scripts/kubernetesdev/values.yaml b/scripts/kubernetesdev/values.yaml new file mode 100644 index 0000000..ccde8e9 --- /dev/null +++ b/scripts/kubernetesdev/values.yaml @@ -0,0 +1,33 @@ +--- + +# Postgres credentials are stored in a secret mounts by PostgreSQL and Partition manager +postgresDatabase: development +postgresUser: root +postgresPassword: hackme + +postgresql-partition-manager: + image: + repository: postgresql-partition-manager # Override to use local registry + tag: dev # Override to use local tag + + cronjob: + postgresqlUserSecret: + ref: postgres-credentials + key: user + postgresqlPasswordSecret: + ref: postgres-credentials + key: password + + configuration: + debug: true + connection-url: postgres://postgres/development + log-format: text + partitions: + by_date: + schema: public + table: by_date + partitionKey: created_at + interval: yearly + retention: 1 + preProvisioned: 1 + cleanupPolicy: drop From ddd0a97d75c4ba5ed7736061db6b0cf89f38b85d Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 13:50:18 +0200 Subject: [PATCH 25/27] chore(goreleaser): Add Goreleaser configuration --- .goreleaser.yaml | 163 ++++++++++++++++++++++++++++++++++ configs/goreleaser/Dockerfile | 16 ++++ 2 files changed, 179 insertions(+) create mode 100644 .goreleaser.yaml create mode 100644 configs/goreleaser/Dockerfile diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..3f4e006 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,163 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +--- +env: + - BUILD_INFO_PACKAGE_PATH=github.com/qonto/postgresql-partition-manager/internal/infra/build + - DOCKER_REGISTRY=public.ecr.aws/qonto + - DOCKER_IMAGE_NAME=postgresql-partition-manager + +builds: + - env: + - CGO_ENABLED=0 + ldflags: + - '-s' + - '-w' + - '-X "{{ .Env.BUILD_INFO_PACKAGE_PATH }}.Version={{.Version}}"' + - '-X "{{ .Env.BUILD_INFO_PACKAGE_PATH }}.CommitSHA={{.Commit}}"' + - '-X "{{ .Env.BUILD_INFO_PACKAGE_PATH }}.Date={{.Date}}"' + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + +nfpms: + - id: debian + file_name_template: "{{ .ConventionalFileName }}" + homepage: https://github.com/qonto/postgresql-partition-manager + description: Streamline the management of PostgreSQL partitions + maintainer: SRE Team + vendor: Qonto + section: misc + license: MIT + formats: + - deb + recommends: + - postgresql-client + contents: + - src: configs/postgresql-partition-manager/postgresql-partition-manager.yaml + dst: /usr/share/postgresql-partition-manager/postgresql-partition-manager.yaml.sample + type: config + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip + files: + - src: configs/postgresql-partition-manager/postgresql-partition-manager.yaml + dst: postgresql-partition-manager.yaml + +checksum: + name_template: 'checksums.txt' + +snapshot: + name_template: "{{ incpatch .Version }}-next" + +changelog: + sort: asc + use: github + filters: + exclude: + - "^test:" + - "^chore" + - "merge conflict" + - Merge pull request + - Merge remote-tracking branch + - Merge branch + - go mod tidy + groups: + - title: Dependency updates + regexp: '^.*?(feat|fix)\(deps\)!?:.+$' + order: 300 + - title: "New Features" + regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' + order: 100 + - title: "Security updates" + regexp: '^.*?sec(\([[:word:]]+\))??!?:.+$' + order: 150 + - title: "Bug fixes" + regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' + order: 200 + - title: "Documentation updates" + regexp: ^.*?doc(\([[:word:]]+\))??!?:.+$ + order: 400 + - title: "Build process updates" + regexp: ^.*?build(\([[:word:]]+\))??!?:.+$ + order: 400 + - title: Other work + order: 9999 + +dockers: + - image_templates: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-amd64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-amd64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-amd64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-amd64" + dockerfile: configs/goreleaser/Dockerfile + build_flag_templates: + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.description={{ .ProjectName }} + - --label=org.opencontainers.image.url=https://github.com/qonto/postgresql-partition-manager + - --label=org.opencontainers.image.source=https://github.com/qonto/postgresql-partition-manager + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=MIT + - "--pull" + - "--platform=linux/amd64" + use: buildx + - image_templates: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-arm64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-arm64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-arm64" + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-arm64" + dockerfile: configs/goreleaser/Dockerfile + build_flag_templates: + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.description={{ .ProjectName }} + - --label=org.opencontainers.image.url=https://github.com/qonto/postgresql-partition-manager + - --label=org.opencontainers.image.source=https://github.com/qonto/postgresql-partition-manager + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=MIT + - "--pull" + - "--platform=linux/arm64" + use: buildx + goarch: arm64 + +docker_manifests: + - name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:{{ .Tag }}-arm64' + - name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}-arm64' + - name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:v{{ .Major }}.{{ .Minor }}-arm64' + - name_template: '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest' + image_templates: + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-amd64' + - '{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_IMAGE_NAME }}:latest-arm64' + +release: + github: + owner: qonto + name: postgresql-partition-manager + name_template: "v{{.Version}}" + footer: | + **Full Changelog**: https://github.com/qonto/postgresql-partition-manager/compare/{{ .PreviousTag }}...{{ .Tag }} diff --git a/configs/goreleaser/Dockerfile b/configs/goreleaser/Dockerfile new file mode 100644 index 0000000..3c69ef6 --- /dev/null +++ b/configs/goreleaser/Dockerfile @@ -0,0 +1,16 @@ +FROM alpine:3 + +ARG USER=app +ARG HOME=/app + +RUN addgroup -g 1001 -S app \ + && adduser --home /app -u 1001 -S app -G app \ + && mkdir -p /app \ + && chown app:app -R /app + +WORKDIR $HOME +USER $USER + +COPY postgresql-partition-manager /app/ + +ENTRYPOINT ["/app/postgresql-partition-manager"] From 5d1755416bc9a38bf861f0aec4d9324178278857 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 15:11:40 +0200 Subject: [PATCH 26/27] chore(ci): Add checkov --- .checkov.yaml | 7 +++++++ .github/workflows/test.yaml | 24 ++++++++++++++++++++++++ Makefile | 3 +++ 3 files changed, 34 insertions(+) create mode 100644 .checkov.yaml diff --git a/.checkov.yaml b/.checkov.yaml new file mode 100644 index 0000000..4ac0170 --- /dev/null +++ b/.checkov.yaml @@ -0,0 +1,7 @@ +--- +skip-check: + - CKV_SECRET_4 # We have password in our configuration templates + - CKV_DOCKER_2 # We don't define healthcheck for our containers because it's a CLI + +skip-path: + - scripts/kubernetesdev # Exclude kubernetes development environment, mostly for PostgreSQL diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2ffe32b..64ea27e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -155,3 +155,27 @@ jobs: GORELEASER_CURRENT_TAG: 0.0.0 - name: Run Debian package tests run: make debian-test-ci + + checkcov: + permissions: + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Checkov GitHub Action + uses: bridgecrewio/checkov-action@v12 + with: + # This will add both a CLI output to the console and create a results.sarif file + output_format: cli,sarif + output_file_path: console,results.sarif + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v3 + # Results are generated only on a success or failure + # this is required since GitHub by default won't run the next step + # when the previous one has failed. Security checks that do not pass will 'fail'. + # An alternative is to add `continue-on-error: true` to the previous step + # Or 'soft_fail: true' to checkov. + if: success() || failure() + with: + sarif_file: results.sarif diff --git a/Makefile b/Makefile index c58548f..2dace18 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ debian-test-ci: docker build configs/debian/tests -t test docker run -v ./dist/postgresql-partition-manager_0.0.1~next_amd64.deb:/mnt/postgresql-partition-manager.deb test +checkcov: + checkov --directory . + .PHONY: test test: go test -race -v ./... -coverprofile=coverage.txt -covermode atomic From 3bc21c9537ba30e1a781829d961a1c303e5ee369 Mon Sep 17 00:00:00 2001 From: Vincent Mercier Date: Thu, 18 Apr 2024 16:00:18 +0200 Subject: [PATCH 27/27] chore(ci): Disable checkov until project is private Checkcov can only work on public projects or using Github Enterprise licence https://docs.github.com/en/code-security/code-scanning/troubleshooting-code-scanning/advanced-security-must-be-enabled We'll enable it once project will be public --- .github/workflows/test.yaml | 46 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 64ea27e..29121fb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -156,26 +156,26 @@ jobs: - name: Run Debian package tests run: make debian-test-ci - checkcov: - permissions: - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Checkov GitHub Action - uses: bridgecrewio/checkov-action@v12 - with: - # This will add both a CLI output to the console and create a results.sarif file - output_format: cli,sarif - output_file_path: console,results.sarif - - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v3 - # Results are generated only on a success or failure - # this is required since GitHub by default won't run the next step - # when the previous one has failed. Security checks that do not pass will 'fail'. - # An alternative is to add `continue-on-error: true` to the previous step - # Or 'soft_fail: true' to checkov. - if: success() || failure() - with: - sarif_file: results.sarif +# checkcov: +# permissions: +# security-events: write # for github/codeql-action/upload-sarif to upload SARIF results +# actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - name: Checkov GitHub Action +# uses: bridgecrewio/checkov-action@v12 +# with: +# # This will add both a CLI output to the console and create a results.sarif file +# output_format: cli,sarif +# output_file_path: console,results.sarif +# - name: Upload SARIF file +# uses: github/codeql-action/upload-sarif@v3 +# # Results are generated only on a success or failure +# # this is required since GitHub by default won't run the next step +# # when the previous one has failed. Security checks that do not pass will 'fail'. +# # An alternative is to add `continue-on-error: true` to the previous step +# # Or 'soft_fail: true' to checkov. +# if: success() || failure() +# with: +# sarif_file: results.sarif