Timestamptz fix

.github/workflows/linter.yaml

@@ -16,12 +16,12 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.23'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.54
+          version: v1.61
 
   yamllint:
     name: yamllint

.github/workflows/test.yaml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.23'
       - name: Install dependencies
         run: |
           go get .
@@ -167,7 +167,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.23'
       - name: Install dependencies
         run: |
           go get .

.golangci.yml

@@ -1,12 +1,13 @@
 ---
 run:
   concurrency: 4
-  deadline: 2m
+  timeout: 2m
   issues-exit-code: 1
   tests: true
 
 output:
-  format: colored-line-number
+  formats:
+    - format: colored-line-number
   print-issued-lines: true
   print-linter-name: true
 
@@ -34,11 +35,11 @@ linters:
     - gocritic
     - gocyclo
     - godox
-    - goerr113
+    - err113
     - gofmt
     - gofumpt
     - goimports
-    - gomnd
+    - mnd
     - gomodguard
     - goprintffuncname
     - gosec
@@ -81,8 +82,8 @@ linters-settings:
 
 issues:
   exclude-use-default: false
-  max-per-linter: 1024
-  max-same: 1024
+  max-issues-per-linter: 1024
+  max-same-issues: 1024
 
   exclude-rules:
     # Exclude some linters from running on test files

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
       - id: detect-private-key
       - id: check-symlinks
   - repo: https://github.com/golangci/golangci-lint
-    rev: v1.54.2
+    rev: v1.61.0
     hooks:
       - id: golangci-lint
   - repo: https://github.com/Bahjat/pre-commit-golang

README.md

@@ -21,19 +21,21 @@ PPM will process all referenced partitions and exit with a non-zero code if it d
 
 - Support of PostgreSQL 14+
 - Only supports [`RANGE` partition strategy](https://www.postgresql.org/docs/current/ddl-partitioning.html#DDL-PARTITIONING-OVERVIEW-RANGE)
-- The partition key must be a column of `date`, `timestamp`, or `uuid` type
+- Gaps are not allowed in-between partitions
+- The partition key must be a column of `date`, `timestamp`, `timestamptz` or `uuid` type
 - Support `daily`, `weekly`, `monthly`, `quarterly`, and `yearly` partitioning
-- Dates are implemented through UTC timezone
-- Partition names are enforced and not configurable
+- Partition names are not configurable
 
   | Partition interval | Pattern                                   | Example           |
   | ------------------ | ----------------------------------------- | ----------------- |
   | daily              | `<parent_table>_<YYYY>_<DD>_<MM>`         | `logs_2024_06_25` |
-  | weekly             | `<parent_table>_w<week number>`           | `logs_2024_w26`   |
+  | weekly             | `<parent_table>_w<ISO week number>`       | `logs_2024_w26`   |
   | quarterly          | `<parent_table>_<YYYY>_q<quarter number>` | `logs_2024_q1`    |
   | monthly            | `<parent_table>_<YYYY>_<MM>`              | `logs_2024_06`    |
   | yearly             | `<parent_table>_<YYYY>`                   | `logs_2024`       |
 
+Whenever that interval is changed in the configuration, the provisioning may create new partitions shorter than the interval and named `<parent_table>_<YYYYDDMM>_<YYYYDDMM>` to fill the gaps between the existing and new partitions. These gaps can range from one day to the new interval size minus one day.
+
 ## Installation
 
 PPM is available as a Docker image, Debian package, and Binary.

go.mod

@@ -1,14 +1,14 @@
 module github.com/qonto/postgresql-partition-manager
 
-go 1.21.3
+go 1.23.0
 
-toolchain go1.21.9
+toolchain go1.23.2
 
 require (
 	github.com/go-playground/validator/v10 v10.22.0
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgconn v1.14.3
-	github.com/jackc/pgx/v5 v5.6.0
+	github.com/jackc/pgx/v5 v5.7.5
 	github.com/pashagolub/pgxmock/v3 v3.4.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
@@ -29,8 +29,8 @@ require (
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -47,12 +47,12 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/crypto v0.37.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sync v0.13.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -38,12 +38,12 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag=
 github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
-github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
-github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
-github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
-github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
+github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -101,18 +101,18 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

internal/infra/uuid7/uuid7.go

@@ -8,7 +8,7 @@ import (
 	"time"
 )
 
-//nolint:gomnd
+//nolint:mnd
 func FromTime(timestamp time.Time) string {
 	// Convert timestamp to Unix time in milliseconds
 	unixMillis := timestamp.UnixNano() / int64(time.Millisecond)
@@ -17,8 +17,8 @@ func FromTime(timestamp time.Time) string {
 	// Ensure the slice is initially 8 bytes to accommodate the full uint64,
 	// but we'll only use the last 6 bytes for the timestamp
 	timeBytes := make([]byte, 8)
-	binary.BigEndian.PutUint64(timeBytes, uint64(unixMillis))
-	timeBytes = timeBytes[2:] // Keep the last 6 bytes
+	binary.BigEndian.PutUint64(timeBytes, uint64(unixMillis)) //nolint:gosec
+	timeBytes = timeBytes[2:]                                 // Keep the last 6 bytes
 
 	// Generate random bytes for the rest of the UUID (10 bytes to make it a total of 16)
 	randomBytes := make([]byte, 10)

pkg/ppm/bounds.go

@@ -114,7 +114,8 @@ func parseBoundAsUUIDv7(partition postgresql.PartitionResult) (lowerBound, upper
 }
 
 func convertToDateTimeWithoutTimezone(bound time.Time) time.Time {
-	parsedTime, err := time.Parse("2006-01-02 15:04:05", bound.UTC().Format("2006-01-02 15:04:05"))
+	/* Remove the time zone offset without rotating the timestamp to UTC */
+	parsedTime, err := time.Parse("2006-01-02 15:04:05", bound.Format("2006-01-02 15:04:05"))
 	if err != nil {
 		return time.Time{}
 	}

pkg/ppm/bounds_internal_test.go

@@ -45,8 +45,8 @@ func TestParseBounds(t *testing.T) {
 				LowerBound: "2024-01-01 23:30:00-01",
 				UpperBound: "2025-02-03 00:30:00+01",
 			},
-			"2024-01-02T00:30:00Z",
-			"2025-02-02T23:30:00Z",
+			"2024-01-01T23:30:00Z",
+			"2025-02-03T00:30:00Z",
 		},
 		{
 			"UUIDv7 bounds",

scripts/bats/30_provisioning.bats

@@ -364,3 +364,55 @@ EOF
   run list_existing_partitions "unittest" "public" "${TABLE}"
   assert_output "$expected2"
 }
+
+@test "Test a timestamptz key with provisioning crossing a DST transition " {
+  local TABLE="test_tz1"
+  local INTERVAL=weekly
+  local RETENTION=3
+  local PREPROVISIONED=2
+
+  declare -x PGTZ="Europe/Paris"
+
+  create_table_timestamptz_range ${TABLE}
+
+  local CONFIGURATION=$(basic_configuration ${TABLE} weekly created_at $RETENTION $PREPROVISIONED)
+  local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
+
+  PPM_WORK_DATE="2025-03-06" run "$PPM_PROG" run provisioning -c ${CONFIGURATION_FILE}
+  assert_success
+  assert_output --partial "All partitions are correctly provisioned"
+
+  local expected_1=$(cat <<EOF
+public|${TABLE}_2025_w07|2025-02-10 00:00:00+01|2025-02-17 00:00:00+01
+public|${TABLE}_2025_w08|2025-02-17 00:00:00+01|2025-02-24 00:00:00+01
+public|${TABLE}_2025_w09|2025-02-24 00:00:00+01|2025-03-03 00:00:00+01
+public|${TABLE}_2025_w10|2025-03-03 00:00:00+01|2025-03-10 00:00:00+01
+public|${TABLE}_2025_w11|2025-03-10 00:00:00+01|2025-03-17 00:00:00+01
+public|${TABLE}_2025_w12|2025-03-17 00:00:00+01|2025-03-24 00:00:00+01
+EOF
+  )
+
+  run list_existing_partitions "unittest" "public" ${TABLE}
+  assert_output "$expected_1"
+
+  # Now advance one week up to the end of March 2025
+  # The transition to summer time (GMT+1 => GMT+2) occurs at 2025-05-30 02:00 => 03:00
+
+  PPM_WORK_DATE="2025-03-13" run "$PPM_PROG" run provisioning -c ${CONFIGURATION_FILE}
+  assert_success
+  assert_output --partial "All partitions are correctly provisioned"
+
+  local expected_2=$(cat <<EOF
+public|${TABLE}_2025_w07|2025-02-10 00:00:00+01|2025-02-17 00:00:00+01
+public|${TABLE}_2025_w08|2025-02-17 00:00:00+01|2025-02-24 00:00:00+01
+public|${TABLE}_2025_w09|2025-02-24 00:00:00+01|2025-03-03 00:00:00+01
+public|${TABLE}_2025_w10|2025-03-03 00:00:00+01|2025-03-10 00:00:00+01
+public|${TABLE}_2025_w11|2025-03-10 00:00:00+01|2025-03-17 00:00:00+01
+public|${TABLE}_2025_w12|2025-03-17 00:00:00+01|2025-03-24 00:00:00+01
+public|${TABLE}_2025_w13|2025-03-24 00:00:00+01|2025-03-31 00:00:00+02
+EOF
+)
+  run list_existing_partitions "unittest" "public" ${TABLE}
+  assert_output "$expected_2"
+
+}

scripts/bats/test/libs/seeds.bash

@@ -39,6 +39,19 @@ EOQ
   execute_sql "${QUERY}"
 }
 
+create_table_timestamptz_range() {
+  local TABLE="$1"
+
+  read -r -d '' QUERY <<EOQ ||
+  CREATE TABLE ${TABLE} (
+    id              BIGSERIAL,
+    value	    INT,
+    created_at      TIMESTAMPTZ NOT NULL
+  ) PARTITION BY RANGE (created_at);
+EOQ
+  execute_sql "${QUERY}"
+}
+
 generate_configuration_file() {
   local PARTITION_CONFIGURATION=$1
   local CONFIGURATION_TEMPLATE_FILE=configuration/template.yaml
@@ -51,3 +64,19 @@ generate_configuration_file() {
 
   echo $FILENAME
 }
+
+# Return a common configuration
+# Arguments: table interval partition-key retention preprovisioned
+basic_configuration() {
+cat << EOF_conf
+partitions:
+  unittest:
+    schema: public
+    table: $1
+    interval: $2
+    partitionKey: $3
+    cleanupPolicy: drop
+    retention: $4
+    preProvisioned: $5
+EOF_conf
+}