diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 60b24bc..a60fa51 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -16,12 +16,12 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' cache: false - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.54 + version: v1.61 yamllint: name: yamllint diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7df06ed..902807b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' - name: Install dependencies run: | go get . @@ -167,7 +167,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' - name: Install dependencies run: | go get . diff --git a/.golangci.yml b/.golangci.yml index 5ed6719..8ce3224 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,12 +1,13 @@ --- run: concurrency: 4 - deadline: 2m + timeout: 2m issues-exit-code: 1 tests: true output: - format: colored-line-number + formats: + - format: colored-line-number print-issued-lines: true print-linter-name: true @@ -34,11 +35,11 @@ linters: - gocritic - gocyclo - godox - - goerr113 + - err113 - gofmt - gofumpt - goimports - - gomnd + - mnd - gomodguard - goprintffuncname - gosec @@ -81,8 +82,8 @@ linters-settings: issues: exclude-use-default: false - max-per-linter: 1024 - max-same: 1024 + max-issues-per-linter: 1024 + max-same-issues: 1024 exclude-rules: # Exclude some linters from running on test files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03f120c..86944a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: detect-private-key - id: check-symlinks - repo: https://github.com/golangci/golangci-lint - rev: v1.54.2 + rev: v1.61.0 hooks: - id: golangci-lint - repo: https://github.com/Bahjat/pre-commit-golang diff --git a/README.md b/README.md index 628212c..85de221 100644 --- a/README.md +++ b/README.md @@ -21,19 +21,21 @@ PPM will process all referenced partitions and exit with a non-zero code if it d - Support of PostgreSQL 14+ - Only supports [`RANGE` partition strategy](https://www.postgresql.org/docs/current/ddl-partitioning.html#DDL-PARTITIONING-OVERVIEW-RANGE) -- The partition key must be a column of `date`, `timestamp`, or `uuid` type +- Gaps are not allowed in-between partitions +- The partition key must be a column of `date`, `timestamp`, `timestamptz` or `uuid` type - Support `daily`, `weekly`, `monthly`, `quarterly`, and `yearly` partitioning -- Dates are implemented through UTC timezone -- Partition names are enforced and not configurable +- Partition names are not configurable | Partition interval | Pattern | Example | | ------------------ | ----------------------------------------- | ----------------- | | daily | `__
_` | `logs_2024_06_25` | - | weekly | `_w` | `logs_2024_w26` | + | weekly | `_w` | `logs_2024_w26` | | quarterly | `__q` | `logs_2024_q1` | | monthly | `__` | `logs_2024_06` | | yearly | `_` | `logs_2024` | +Whenever that interval is changed in the configuration, the provisioning may create new partitions shorter than the interval and named `__` to fill the gaps between the existing and new partitions. These gaps can range from one day to the new interval size minus one day. + ## Installation PPM is available as a Docker image, Debian package, and Binary. diff --git a/go.mod b/go.mod index 542464d..2c4111b 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module github.com/qonto/postgresql-partition-manager -go 1.21.3 +go 1.23.0 -toolchain go1.21.9 +toolchain go1.23.2 require ( github.com/go-playground/validator/v10 v10.22.0 github.com/google/uuid v1.6.0 github.com/jackc/pgconn v1.14.3 - github.com/jackc/pgx/v5 v5.6.0 + github.com/jackc/pgx/v5 v5.7.5 github.com/pashagolub/pgxmock/v3 v3.4.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 @@ -29,8 +29,8 @@ require ( github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -47,12 +47,12 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 490aea2..2637548 100644 --- a/go.sum +++ b/go.sum @@ -38,12 +38,12 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -101,18 +101,18 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/infra/uuid7/uuid7.go b/internal/infra/uuid7/uuid7.go index 2cff57d..a55dfab 100644 --- a/internal/infra/uuid7/uuid7.go +++ b/internal/infra/uuid7/uuid7.go @@ -8,7 +8,7 @@ import ( "time" ) -//nolint:gomnd +//nolint:mnd func FromTime(timestamp time.Time) string { // Convert timestamp to Unix time in milliseconds unixMillis := timestamp.UnixNano() / int64(time.Millisecond) @@ -17,8 +17,8 @@ func FromTime(timestamp time.Time) string { // Ensure the slice is initially 8 bytes to accommodate the full uint64, // but we'll only use the last 6 bytes for the timestamp timeBytes := make([]byte, 8) - binary.BigEndian.PutUint64(timeBytes, uint64(unixMillis)) - timeBytes = timeBytes[2:] // Keep the last 6 bytes + binary.BigEndian.PutUint64(timeBytes, uint64(unixMillis)) //nolint:gosec + timeBytes = timeBytes[2:] // Keep the last 6 bytes // Generate random bytes for the rest of the UUID (10 bytes to make it a total of 16) randomBytes := make([]byte, 10) diff --git a/pkg/ppm/bounds.go b/pkg/ppm/bounds.go index 99defbf..ea08a1e 100644 --- a/pkg/ppm/bounds.go +++ b/pkg/ppm/bounds.go @@ -114,7 +114,8 @@ func parseBoundAsUUIDv7(partition postgresql.PartitionResult) (lowerBound, upper } func convertToDateTimeWithoutTimezone(bound time.Time) time.Time { - parsedTime, err := time.Parse("2006-01-02 15:04:05", bound.UTC().Format("2006-01-02 15:04:05")) + /* Remove the time zone offset without rotating the timestamp to UTC */ + parsedTime, err := time.Parse("2006-01-02 15:04:05", bound.Format("2006-01-02 15:04:05")) if err != nil { return time.Time{} } diff --git a/pkg/ppm/bounds_internal_test.go b/pkg/ppm/bounds_internal_test.go index afdda04..beda2ee 100644 --- a/pkg/ppm/bounds_internal_test.go +++ b/pkg/ppm/bounds_internal_test.go @@ -45,8 +45,8 @@ func TestParseBounds(t *testing.T) { LowerBound: "2024-01-01 23:30:00-01", UpperBound: "2025-02-03 00:30:00+01", }, - "2024-01-02T00:30:00Z", - "2025-02-02T23:30:00Z", + "2024-01-01T23:30:00Z", + "2025-02-03T00:30:00Z", }, { "UUIDv7 bounds", diff --git a/scripts/bats/30_provisioning.bats b/scripts/bats/30_provisioning.bats index a2e9fb8..c36fae5 100644 --- a/scripts/bats/30_provisioning.bats +++ b/scripts/bats/30_provisioning.bats @@ -364,3 +364,55 @@ EOF run list_existing_partitions "unittest" "public" "${TABLE}" assert_output "$expected2" } + +@test "Test a timestamptz key with provisioning crossing a DST transition " { + local TABLE="test_tz1" + local INTERVAL=weekly + local RETENTION=3 + local PREPROVISIONED=2 + + declare -x PGTZ="Europe/Paris" + + create_table_timestamptz_range ${TABLE} + + local CONFIGURATION=$(basic_configuration ${TABLE} weekly created_at $RETENTION $PREPROVISIONED) + local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}") + + PPM_WORK_DATE="2025-03-06" run "$PPM_PROG" run provisioning -c ${CONFIGURATION_FILE} + assert_success + assert_output --partial "All partitions are correctly provisioned" + + local expected_1=$(cat < GMT+2) occurs at 2025-05-30 02:00 => 03:00 + + PPM_WORK_DATE="2025-03-13" run "$PPM_PROG" run provisioning -c ${CONFIGURATION_FILE} + assert_success + assert_output --partial "All partitions are correctly provisioned" + + local expected_2=$(cat <