From da42314205d125bf761cdd567350f3f891d4123f Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Tue, 4 Aug 2026 22:43:48 +0200 Subject: [PATCH] Implement table partition management --- README.md | 128 ++++++++- lib/code0/zero_track/database/migration.rb | 1 + .../migration_helpers/table_partitioning.rb | 106 ++++++++ .../partitioning/partition_manager.rb | 144 ++++++++++ .../partitioning/partitioned_table.rb | 33 +++ .../postgres_detached_partition.rb | 26 ++ .../partitioning/postgres_partition.rb | 24 ++ .../postgres_partitioned_table.rb | 30 ++ .../database/partitioning/strategy/base.rb | 67 +++++ .../database/partitioning/strategy/daily.rb | 82 ++++++ .../database/partitioning/strategy/monthly.rb | 82 ++++++ .../database/partitioning/time_partition.rb | 94 +++++++ .../zero_track/database/schema_cleaner.rb | 29 ++ lib/code0/zero_track/railtie.rb | 5 + .../partitioning/partition_manager_spec.rb | 257 ++++++++++++++++++ .../partitioning/partitioned_table_spec.rb | 65 +++++ .../partitioning/strategy/base_spec.rb | 64 +++++ .../partitioning/strategy/daily_spec.rb | 152 +++++++++++ .../partitioning/strategy/monthly_spec.rb | 155 +++++++++++ .../partitioning/time_partition_spec.rb | 194 +++++++++++++ .../database/schema_cleaner_spec.rb | 225 +++++++++++++++ 21 files changed, 1962 insertions(+), 1 deletion(-) create mode 100644 lib/code0/zero_track/database/migration_helpers/table_partitioning.rb create mode 100644 lib/code0/zero_track/database/partitioning/partition_manager.rb create mode 100644 lib/code0/zero_track/database/partitioning/partitioned_table.rb create mode 100644 lib/code0/zero_track/database/partitioning/postgres_detached_partition.rb create mode 100644 lib/code0/zero_track/database/partitioning/postgres_partition.rb create mode 100644 lib/code0/zero_track/database/partitioning/postgres_partitioned_table.rb create mode 100644 lib/code0/zero_track/database/partitioning/strategy/base.rb create mode 100644 lib/code0/zero_track/database/partitioning/strategy/daily.rb create mode 100644 lib/code0/zero_track/database/partitioning/strategy/monthly.rb create mode 100644 lib/code0/zero_track/database/partitioning/time_partition.rb create mode 100644 spec/code0/zero_track/database/partitioning/partition_manager_spec.rb create mode 100644 spec/code0/zero_track/database/partitioning/partitioned_table_spec.rb create mode 100644 spec/code0/zero_track/database/partitioning/strategy/base_spec.rb create mode 100644 spec/code0/zero_track/database/partitioning/strategy/daily_spec.rb create mode 100644 spec/code0/zero_track/database/partitioning/strategy/monthly_spec.rb create mode 100644 spec/code0/zero_track/database/partitioning/time_partition_spec.rb create mode 100644 spec/code0/zero_track/database/schema_cleaner_spec.rb diff --git a/README.md b/README.md index 1a49660..221222e 100644 --- a/README.md +++ b/README.md @@ -65,4 +65,130 @@ can be filled with the correct entries when the schema is loaded from the schema This approach is prone to git conflicts, so you can switch to a file based persistence with `config.zero_track.active_record.schema_migrations = true`. Instead of an `INSERT INTO` in -the `db/structure.sql`, this mode creates files in the `db/schema_migrations` directory. \ No newline at end of file +the `db/structure.sql`, this mode creates files in the `db/schema_migrations` directory. + +### Table Partitioning + +This gem provides automated time-based table partitioning for PostgreSQL using range partitioning. +Partitions are dynamically created, detached, and dropped based on a configurable strategy. + +#### Configuration + +```ruby +config.zero_track.db_partitioning.dynamic_partition_schema = 'partitions_dynamic' # default +config.zero_track.db_partitioning.base_ar_class = 'ActiveRecord::Base' # default +``` + +- `dynamic_partition_schema` — The PostgreSQL schema where dynamic partitions are stored. +- `base_ar_class` — The ActiveRecord base class used for the internal partitioning models. + +#### Migration Helpers + +Include the migration helpers by inheriting from `Code0::ZeroTrack::Database::Migration[1.0]` (or the +appropriate version). The following methods become available: + +`create_partition_by_date_table(table_name, partition_column:, **options, &block)` creates a table +partitioned by range on the given column. It automatically sets up a composite primary key +of `(id, partition_column)`. + +`create_dynamic_partition_schema` / `drop_dynamic_partition_schema` creates or drops the schema +used for storing dynamic partitions. + +`create_partitioning_views` / `drop_partitioning_views` creates or drops the PostgreSQL views +(`postgres_partitioned_tables`, `postgres_partitions`, `postgres_detached_partitions`) that the +partition manager uses to inspect existing partitions. + +Example migration: + +```ruby +class CreatePartitionedEvents < Code0::ZeroTrack::Database::Migration[1.0] + def change + create_dynamic_partition_schema + create_partitioning_views + + create_partition_by_date_table :events, partition_column: :created_at do |t| + t.text :name, null: false + t.timestamps_with_timezone null: false + end + end +end +``` + +#### Defining a Partitioned Model + +Include `Code0::ZeroTrack::Database::Partitioning::PartitionedTable` in your model and declare +the partitioning strategy: + +```ruby +class Event < ApplicationRecord + include Code0::ZeroTrack::Database::Partitioning::PartitionedTable + + partition_by :created_at, strategy: :monthly, retain_for: 12.months +end +``` + +Available strategies: `:daily` and `:monthly`. + +Options passed to `partition_by`: + +| Option | Description | Default | +|--------|-------------|---------| +| `strategy` | `:daily` or `:monthly` | *required* | +| `headroom` | How far ahead to pre-create partitions | 30 days (daily) / 6 months (monthly) | +| `retain_for` | How long to keep partitions before detaching (enables retention) | `nil` (disabled) | +| `retain_detached_for` | How long to keep detached partitions before dropping | 7 days | + +#### Partition Manager + +`Code0::ZeroTrack::Database::Partitioning::PartitionManager` handles the lifecycle of partitions. + +Register models for automatic partition management: + +```ruby +Code0::ZeroTrack::Database::Partitioning::PartitionManager.register_model(Event) +Code0::ZeroTrack::Database::Partitioning::PartitionManager.register_model(EventDetail) +``` + +Then synchronize all registered models. The gem does not invoke this automatically — your +application is responsible for calling it at the appropriate time (e.g., in a recurring +background job or during deployment): + +```ruby +Code0::ZeroTrack::Database::Partitioning::PartitionManager.sync_all_partitions +``` + +Or manage a single model: + +```ruby +manager = Code0::ZeroTrack::Database::Partitioning::PartitionManager.new(Event) +manager.sync_partitions +``` + +`sync_all_partitions` first creates partitions for all registered models, then detaches and drops +partitions for all models in reverse registration order. This two-phase approach ensures that +parent partitions exist before child partitions are created, and child partitions are cleaned up +before their parents. + +`sync_partitions` performs three operations in order for a single model: +1. **Create** — Creates and attaches new partitions to cover the desired range (up to the configured headroom). +2. **Detach** — Detaches partitions that fall outside the desired range (when retention is enabled). +3. **Drop** — Drops detached partitions that have been detached longer than `retain_detached_for`. + +All operations are protected by a PostgreSQL advisory lock per table to ensure safe concurrent execution. + +**Important:** When tables have foreign key relationships, registration order and retention +configuration matter: + +- **Registration order** — Register parent tables before child tables. `sync_all_partitions` + creates partitions in registration order and detaches/drops them in reverse order. This + prevents foreign key violations by ensuring parent partitions exist when child partitions + are attached, and child partitions are removed before their referenced parent partitions. +- **Retention alignment** — A child table's `retain_for` must be less than or equal to the + parent table's `retain_for`. If a child retains partitions longer than its parent, dropping + the parent partition will fail because the child's foreign key still references it. + +#### Schema Cleaner Integration + +When `config.zero_track.active_record.schema_cleaner = true`, dynamic partition objects in the +`dynamic_partition_schema` are automatically removed from `db/structure.sql` during schema dumps. +This prevents schema drift caused by partition rotation. \ No newline at end of file diff --git a/lib/code0/zero_track/database/migration.rb b/lib/code0/zero_track/database/migration.rb index 26be5e7..5303239 100644 --- a/lib/code0/zero_track/database/migration.rb +++ b/lib/code0/zero_track/database/migration.rb @@ -11,6 +11,7 @@ class V1_0 < ::ActiveRecord::Migration[7.1] include Database::MigrationHelpers::IndexHelpers include Database::MigrationHelpers::RemoveColumnEnhancements include Database::MigrationHelpers::TableEnhancements + include Database::MigrationHelpers::TablePartitioning end # rubocop:enable Naming/ClassAndModuleCamelCase diff --git a/lib/code0/zero_track/database/migration_helpers/table_partitioning.rb b/lib/code0/zero_track/database/migration_helpers/table_partitioning.rb new file mode 100644 index 0000000..ba42bf9 --- /dev/null +++ b/lib/code0/zero_track/database/migration_helpers/table_partitioning.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module MigrationHelpers + module TablePartitioning + def create_partition_by_date_table(table_name, partition_column:, **options, &block) + options[:options] = "PARTITION BY RANGE (#{quote_column_name(partition_column)})" + options[:id] = false + + create_table(table_name, **options) do |t| + t.bigserial :id, null: false + + block.call(t) + end + + reversible do |dir| + dir.up do + execute <<~SQL.squish + ALTER TABLE #{quote_table_name(table_name)} + ADD PRIMARY KEY (#{quote_column_name(:id)}, #{quote_column_name(partition_column)}) + SQL + end + end + end + + def create_dynamic_partition_schema + schema = quote_table_name(Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema) + execute "CREATE SCHEMA #{schema}" + end + + def drop_dynamic_partition_schema + schema = quote_table_name(Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema) + execute "DROP SCHEMA #{schema}" + end + + def create_partitioning_views + dynamic_schema = Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema + + execute <<-SQL.squish + CREATE OR REPLACE VIEW postgres_partitioned_tables AS + SELECT c.oid::regclass::text AS identifier, + c.oid, + n.nspname AS schema, + c.relname AS name, + CASE p.partstrat + WHEN 'l' THEN 'list' + WHEN 'r' THEN 'range' + WHEN 'h' THEN 'hash' + END AS strategy, + pg_get_partkeydef(c.oid) AS partition_key + FROM pg_partitioned_table p + JOIN pg_class c ON c.oid = p.partrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema(); + SQL + + execute <<-SQL.squish + CREATE OR REPLACE VIEW postgres_partitions AS + SELECT c.oid::regclass::text AS identifier, + c.oid, + n.nspname AS schema, + c.relname AS name, + i.inhparent::regclass::text AS parent_identifier, + pg_get_expr(c.relpartbound, c.oid) AS condition, + obj_description(c.oid) AS comment, + i.inhrelid IS NOT NULL AS attached + FROM pg_class c + LEFT JOIN pg_inherits i ON c.oid = i.inhrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relispartition + AND c.relkind = 'r' + AND n.nspname IN (current_schema(), #{quote(dynamic_schema)}); + SQL + + execute <<-SQL.squish + CREATE OR REPLACE VIEW postgres_detached_partitions AS + SELECT c.oid::regclass::text AS identifier, + c.oid, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid)::jsonb ->> 'table' AS parent_identifier, + (obj_description(c.oid)::jsonb ->> 'detached_at')::timestamptz AS detached_at + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind = 'r' + AND n.nspname = #{quote(dynamic_schema)} + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid + ) + AND obj_description(c.oid)::jsonb ? 'table' + AND obj_description(c.oid)::jsonb ? 'detached_at'; + SQL + end + + def drop_partitioning_views + execute 'DROP VIEW IF EXISTS postgres_detached_partitions' + execute 'DROP VIEW IF EXISTS postgres_partitions' + execute 'DROP VIEW IF EXISTS postgres_partitioned_tables' + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/partition_manager.rb b/lib/code0/zero_track/database/partitioning/partition_manager.rb new file mode 100644 index 0000000..7c8338c --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/partition_manager.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +require 'zlib' + +module Code0 + module ZeroTrack + module Database + module Partitioning + class PartitionManager + include Loggable + + cattr_accessor :models + self.models = [] + + def self.register_model(clazz) + models << clazz + end + + def self.sync_all_partitions + models.each do |model| + new(model).create_partitions + end + + models.reverse_each do |model| + manager = new(model) + manager.detach_partitions + manager.drop_partitions + end + end + + attr_reader :model + + def initialize(model) + if model.try(:partitioning_strategy).nil? + raise ArgumentError, "Model #{model} not configured for partitioning" + end + + @model = model + end + + def sync_partitions + create_partitions + + detach_partitions + + drop_partitions + end + + def create_partitions + with_lock do |connection| + model.partitioning_strategy.partitions_to_create.each do |partition| + create_partition(partition, connection) + attach_partition(partition, connection) + end + end + end + + def detach_partitions + with_lock do |connection| + model.partitioning_strategy.partitions_to_detach.each do |partition| + detach_partition(partition, connection) + end + end + end + + def drop_partitions + with_lock do |connection| + model.partitioning_strategy.partitions_to_drop.each do |detached_partition| + drop_partition(detached_partition, connection) + end + end + end + + private + + def create_partition(partition, connection) + connection.execute(partition.to_create_sql(connection)) + logger.info( + message: 'Created new partition', + table_name: partition.model.table_name, + partition_name: partition.partition_name + ) + end + + def attach_partition(partition, connection) + connection.execute(partition.to_attach_sql(connection)) + logger.info( + message: 'Attached partition', + table_name: partition.model.table_name, + partition_name: partition.partition_name + ) + end + + def detach_partition(partition, connection) + connection.execute(partition.to_detach_sql(connection)) + + partition_comment = connection.quote({ table: model.table_name, detached_at: Time.current.iso8601 }.to_json) + fully_qualified_partition = partition.fully_qualified_partition(connection) + connection.execute("COMMENT ON TABLE #{fully_qualified_partition} IS #{partition_comment}") + + logger.info( + message: 'Detached partition', + table_name: partition.model.table_name, + partition_name: partition.partition_name + ) + end + + def drop_partition(detached_partition, connection) + schema_name = connection.quote_table_name(detached_partition.schema) + partition_name = connection.quote_table_name(detached_partition.name) + qualified_name = "#{schema_name}.#{partition_name}" + connection.execute("DROP TABLE #{qualified_name}") + + logger.info( + message: 'Dropped partition', + table_name: detached_partition.parent_identifier, + partition_name: detached_partition.name + ) + end + + def with_lock + lock_key = lock_key_for(model.table_name) + + with_connection do |connection| + connection.transaction do + connection.execute("SELECT pg_advisory_xact_lock(#{lock_key})") + yield connection + end + end + end + + def lock_key_for(table_name) + namespace = 'zero_track:partition_sync' + Zlib.crc32("#{namespace}:#{table_name}") + end + + def with_connection(&block) + model.with_connection(&block) + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/partitioned_table.rb b/lib/code0/zero_track/database/partitioning/partitioned_table.rb new file mode 100644 index 0000000..0996330 --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/partitioned_table.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + module PartitionedTable + extend ActiveSupport::Concern + + PARTITIONING_STRATEGIES = { + daily: Partitioning::Strategy::Daily, + monthly: Partitioning::Strategy::Monthly, + }.freeze + + class_methods do + attr_reader :partitioning_strategy + + def partition_by(column, strategy:, **kwargs) + raise(ArgumentError, 'Table is already partitioned') unless partitioning_strategy.nil? + + strategy_class = PARTITIONING_STRATEGIES[strategy] || raise( + ArgumentError, + "Unknown partitioning strategy: #{strategy}" + ) + + @partitioning_strategy = strategy_class.new(self, column, **kwargs) + end + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/postgres_detached_partition.rb b/lib/code0/zero_track/database/partitioning/postgres_detached_partition.rb new file mode 100644 index 0000000..f73e9aa --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/postgres_detached_partition.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + class PostgresDetachedPartition < Rails.application.config.zero_track.db_partitioning.base_ar_class.constantize + self.table_name = 'postgres_detached_partitions' + self.primary_key = 'identifier' + + def readonly? + true + end + + scope :detached_before, ->(timestamp) { where(detached_at: ..timestamp) } + + belongs_to :postgres_partitioned_table, + class_name: 'Code0::ZeroTrack::Database::Partitioning::PostgresPartitionedTable', + foreign_key: 'parent_identifier', + primary_key: 'identifier', + inverse_of: :postgres_detached_partitions + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/postgres_partition.rb b/lib/code0/zero_track/database/partitioning/postgres_partition.rb new file mode 100644 index 0000000..ff1287e --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/postgres_partition.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + class PostgresPartition < Rails.application.config.zero_track.db_partitioning.base_ar_class.constantize + self.table_name = 'postgres_partitions' + self.primary_key = 'identifier' + + def readonly? + true + end + + belongs_to :postgres_partitioned_table, + class_name: 'Code0::ZeroTrack::Database::Partitioning::PostgresPartitionedTable', + foreign_key: 'parent_identifier', + primary_key: 'identifier', + inverse_of: :postgres_partitions + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/postgres_partitioned_table.rb b/lib/code0/zero_track/database/partitioning/postgres_partitioned_table.rb new file mode 100644 index 0000000..f90f669 --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/postgres_partitioned_table.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + class PostgresPartitionedTable < Rails.application.config.zero_track.db_partitioning.base_ar_class.constantize + self.table_name = 'postgres_partitioned_tables' + self.primary_key = 'identifier' + + def readonly? + true + end + + has_many :postgres_partitions, + class_name: 'Code0::ZeroTrack::Database::Partitioning::PostgresPartition', + foreign_key: 'parent_identifier', + primary_key: 'identifier', + inverse_of: :postgres_partitioned_table + + has_many :postgres_detached_partitions, + class_name: 'Code0::ZeroTrack::Database::Partitioning::PostgresDetachedPartition', + foreign_key: 'parent_identifier', + primary_key: 'identifier', + inverse_of: :postgres_partitioned_table + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/strategy/base.rb b/lib/code0/zero_track/database/partitioning/strategy/base.rb new file mode 100644 index 0000000..1a14646 --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/strategy/base.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + module Strategy + class Base + attr_reader :model, :partitioning_column, :headroom, :retain_for, :retain_detached_for + + def initialize( + model, + partitioning_column, + headroom: default_headroom, + retain_for: nil, + retain_detached_for: 7.days + ) + @model = model + @partitioning_column = partitioning_column + @headroom = headroom + @retain_for = retain_for + @retain_detached_for = retain_detached_for + end + + def current_partitions + raise NotImplementedError + end + + def desired_partitions + raise NotImplementedError + end + + def oldest_active_date + raise NotImplementedError + end + + def partition_name(lower_bound) + raise NotImplementedError + end + + def default_headroom + raise NotImplementedError + end + + def partitions_to_create + desired_partitions - current_partitions + end + + def partitions_to_detach + current_partitions - desired_partitions + end + + def partitions_to_drop + PostgresPartitionedTable.find_by(identifier: model.table_name) + .postgres_detached_partitions + .detached_before(retain_detached_for.ago) + end + + def retention_enabled? + retain_for.present? + end + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/strategy/daily.rb b/lib/code0/zero_track/database/partitioning/strategy/daily.rb new file mode 100644 index 0000000..04859d7 --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/strategy/daily.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + module Strategy + class Daily < Base + include Loggable + + PARTITION_SUFFIX = '%Y%m%d' + + def current_partitions + partitioned_table = PostgresPartitionedTable.find_by(identifier: model.table_name) + + if partitioned_table.nil? + logger.warn(message: 'Failed to find partitioned table', identifier: model.table_name) + return [] + end + + partitioned_table.postgres_partitions.map do |partition| + TimePartition.from_sql(model, partition.name, partition.condition) + end + end + + def desired_partitions + partitions = [] + + min_date, max_date = desired_range + + while min_date < max_date + next_date = min_date + 1.day + + partitions << TimePartition.new( + model, + min_date, + next_date, + partition_name: partition_name(min_date) + ) + + min_date = next_date + end + + partitions + end + + def desired_range + if retention_enabled? + min_date = oldest_active_date + else + first_partition = current_partitions.min + + min_date = first_partition.from || first_partition.to if first_partition + min_date ||= Date.current + end + + min_date = min_date.beginning_of_day.to_date + + max_date = Date.current + 1.day + headroom + + [min_date, max_date] + end + + def oldest_active_date + retain_for.ago.beginning_of_day.to_date + end + + def partition_name(lower_bound) + suffix = lower_bound&.strftime(PARTITION_SUFFIX) || '00000000' + + "#{model.table_name}_#{suffix}" + end + + def default_headroom + 30.days + end + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/strategy/monthly.rb b/lib/code0/zero_track/database/partitioning/strategy/monthly.rb new file mode 100644 index 0000000..9cb350d --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/strategy/monthly.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + module Strategy + class Monthly < Base + include Loggable + + PARTITION_SUFFIX = '%Y%m' + + def current_partitions + partitioned_table = PostgresPartitionedTable.find_by(identifier: model.table_name) + + if partitioned_table.nil? + logger.warn(message: 'Failed to find partitioned table', identifier: model.table_name) + return [] + end + + partitioned_table.postgres_partitions.map do |partition| + TimePartition.from_sql(model, partition.name, partition.condition) + end + end + + def desired_partitions + partitions = [] + + min_date, max_date = desired_range + + while min_date < max_date + next_date = min_date.next_month + + partitions << TimePartition.new( + model, + min_date, + next_date, + partition_name: partition_name(min_date) + ) + + min_date = next_date + end + + partitions + end + + def desired_range + if retention_enabled? + min_date = oldest_active_date + else + first_partition = current_partitions.min + + min_date = first_partition.from || first_partition.to if first_partition + min_date ||= Date.current + end + + min_date = min_date.beginning_of_month + + max_date = Date.current.end_of_month + headroom + + [min_date, max_date] + end + + def oldest_active_date + retain_for.ago.beginning_of_month.to_date + end + + def partition_name(lower_bound) + suffix = lower_bound&.strftime(PARTITION_SUFFIX) || '000000' + + "#{model.table_name}_#{suffix}" + end + + def default_headroom + 6.months + end + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/partitioning/time_partition.rb b/lib/code0/zero_track/database/partitioning/time_partition.rb new file mode 100644 index 0000000..d01d21e --- /dev/null +++ b/lib/code0/zero_track/database/partitioning/time_partition.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Code0 + module ZeroTrack + module Database + module Partitioning + class TimePartition + include Comparable + + def self.from_sql(table, partition_name, definition) + matches = definition.match(/FOR VALUES FROM \('?(?[^)']+)'?\) TO \('?(?[^)']+)'?\)/) + + raise ArgumentError, "Unknown partition definition: #{definition}" unless matches + + raise NotImplementedError, 'MAXVALUE as upper bound is not supported' if matches[:to] == 'MAXVALUE' + + from = matches[:from] == 'MINVALUE' ? nil : matches[:from] + to = matches[:to] + + new(table, from, to, partition_name: partition_name) + end + + attr_reader :model, :from, :to, :partition_name + + def initialize(model, from, to, partition_name:) + @model = model + @from = date_or_nil(from) + @to = date_or_nil(to) + @partition_name = partition_name + end + + def ==(other) + model == other.model && partition_name == other.partition_name && from == other.from && to == other.to + end + alias eql? == + + def hash + [model, partition_name, from, to].hash + end + + def <=>(other) + return if model != other.model + + partition_name <=> other.partition_name + end + + def to_create_sql(connection) + <<~SQL.squish + CREATE TABLE IF NOT EXISTS #{fully_qualified_partition(connection)} + (LIKE #{connection.quote_table_name(model.table_name)} INCLUDING ALL) + SQL + end + + def to_attach_sql(connection) + from_sql = from ? connection.quote(from.to_date.iso8601) : 'MINVALUE' + to_sql = connection.quote(to.to_date.iso8601) + + <<~SQL.squish + ALTER TABLE #{connection.quote_table_name(model.table_name)} + ATTACH PARTITION #{fully_qualified_partition(connection)} + FOR VALUES FROM (#{from_sql}) TO (#{to_sql}) + SQL + end + + def to_detach_sql(connection) + <<~SQL.squish + ALTER TABLE #{connection.quote_table_name(model.table_name)} + DETACH PARTITION #{fully_qualified_partition(connection)} + SQL + end + + def fully_qualified_partition(connection) + format( + '%s.%s', + schema: connection.quote_table_name( + Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema + ), + partition: connection.quote_table_name(partition_name) + ) + end + + private + + def date_or_nil(obj) + return unless obj + return obj if obj.is_a?(Date) + + Date.parse(obj) + end + end + end + end + end +end diff --git a/lib/code0/zero_track/database/schema_cleaner.rb b/lib/code0/zero_track/database/schema_cleaner.rb index 8cb4e0e..2601518 100644 --- a/lib/code0/zero_track/database/schema_cleaner.rb +++ b/lib/code0/zero_track/database/schema_cleaner.rb @@ -38,6 +38,10 @@ def clean(io) 'CREATE EXTENSION IF NOT EXISTS \1;' ) + # Remove dynamic partition objects that are managed automatically at runtime. + # These would cause schema drift on every partition rotation if left in the dump. + remove_dynamic_partitions!(structure) + structure.gsub!(/\n{3,}/, "\n\n") io << structure.strip @@ -45,6 +49,31 @@ def clean(io) nil end + + private + + def dynamic_partition_schema + Rails.application.config.zero_track.db_partitioning.dynamic_partition_schema + end + + def remove_dynamic_partitions!(structure) + schema = Regexp.escape(dynamic_partition_schema) + + # Remove CREATE TABLE . (...); + structure.gsub!(/^CREATE TABLE #{schema}\.\S+\s*\(.*?\);\n/m, '') + + # Remove ALTER TABLE ... ATTACH PARTITION . ...; + structure.gsub!(/^ALTER TABLE .+ ATTACH PARTITION #{schema}\.\S+.*?;\n/, '') + + # Remove ALTER TABLE ONLY . ...; + structure.gsub!(/^ALTER TABLE ONLY #{schema}\.\S+\n.*?;\n/m, '') + + # Remove CREATE [UNIQUE] INDEX ... ON . ...; + structure.gsub!(/^CREATE (?:UNIQUE )?INDEX \S+ ON #{schema}\.\S+.*?;\n/m, '') + + # Remove ALTER INDEX ... ATTACH PARTITION .; + structure.gsub!(/^ALTER INDEX \S+ ATTACH PARTITION #{schema}\.\S+;\n/, '') + end end end end diff --git a/lib/code0/zero_track/railtie.rb b/lib/code0/zero_track/railtie.rb index d5dd4e3..ab88905 100644 --- a/lib/code0/zero_track/railtie.rb +++ b/lib/code0/zero_track/railtie.rb @@ -4,11 +4,16 @@ module Code0 module ZeroTrack class Railtie < ::Rails::Railtie config.zero_track = ActiveSupport::OrderedOptions.new + config.zero_track.active_record = ActiveSupport::OrderedOptions.new config.zero_track.active_record.timestamps = false config.zero_track.active_record.schema_migrations = false config.zero_track.active_record.schema_cleaner = false + config.zero_track.db_partitioning = ActiveSupport::OrderedOptions.new + config.zero_track.db_partitioning.dynamic_partition_schema = 'partitions_dynamic' + config.zero_track.db_partitioning.base_ar_class = 'ActiveRecord::Base' + rake_tasks do path = File.expand_path(__dir__) Dir.glob("#{path}/../../tasks/**/*.rake").each { |f| load f } diff --git a/spec/code0/zero_track/database/partitioning/partition_manager_spec.rb b/spec/code0/zero_track/database/partitioning/partition_manager_spec.rb new file mode 100644 index 0000000..d23571a --- /dev/null +++ b/spec/code0/zero_track/database/partitioning/partition_manager_spec.rb @@ -0,0 +1,257 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/VerifiedDoubles -- AR models require a DB connection for verification. +RSpec.describe Code0::ZeroTrack::Database::Partitioning::PartitionManager do + include ActiveSupport::Testing::TimeHelpers + + let(:model) do + model = double('Model', table_name: 'events') # rubocop:disable RSpec/VerifiedDoubles + allow(model).to receive(:try).with(:partitioning_strategy).and_return(partitioning_strategy) + allow(model).to receive(:partitioning_strategy).and_return(partitioning_strategy) + allow(model).to receive(:with_connection).and_yield(connection) + model + end + + let(:partitioning_strategy) { double('Strategy') } + + let(:connection) do + connection = double('Connection') + allow(connection).to receive(:execute) + allow(connection).to receive(:transaction).and_yield + allow(connection).to receive(:quote_table_name) { |name| "\"#{name}\"" } + allow(connection).to receive(:quote) { |value| "'#{value}'" } + connection + end + + let(:rails_logger) { instance_double(ActiveSupport::Logger, info: nil, debug: nil, warn: nil, error: nil) } + + before do + allow(Rails).to receive(:logger).and_return(rails_logger) + allow(Rails.application.config.zero_track.db_partitioning).to receive(:dynamic_partition_schema) + .and_return('partitions_dynamic') + end + + describe '#initialize' do + it 'raises ArgumentError if model has no partitioning_strategy' do + model_without_strategy = double('Model', table_name: 'bad') + allow(model_without_strategy).to receive(:try).with(:partitioning_strategy).and_return(nil) + + expect do + described_class.new(model_without_strategy) + end.to raise_error(ArgumentError, /not configured for partitioning/) + end + end + + describe '#sync_partitions' do + it 'creates, detaches, and drops partitions' do + partition_to_create = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-03-01', '2023-04-01', partition_name: 'events_202303' + ) + partition_to_detach = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2022-01-01', '2022-02-01', partition_name: 'events_202201' + ) + partition_to_drop = double('DetachedPartition', schema: 'partitions_dynamic', + name: 'events_202101', + parent_identifier: 'events') + + allow(partitioning_strategy).to receive_messages( + partitions_to_create: [partition_to_create], + partitions_to_detach: [partition_to_detach], + partitions_to_drop: [partition_to_drop] + ) + + manager = described_class.new(model) + manager.sync_partitions + + executed_sql = [] + expect(connection).to have_received(:execute).at_least(:once) do |sql| + executed_sql << sql + end + + expect(executed_sql).to include(a_string_matching(/CREATE TABLE IF NOT EXISTS.*events_202303/)) + expect(executed_sql).to include(a_string_matching(/ATTACH PARTITION.*events_202303/)) + expect(executed_sql).to include(a_string_matching(/DETACH PARTITION.*events_202201/)) + expect(executed_sql).to include(a_string_matching(/DROP TABLE.*events_202101/)) + end + end + + describe '#create_partitions' do + it 'executes CREATE TABLE and ATTACH PARTITION for each partition' do + partition = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-01-01', '2023-02-01', partition_name: 'events_202301' + ) + allow(partitioning_strategy).to receive(:partitions_to_create).and_return([partition]) + + manager = described_class.new(model) + manager.create_partitions + + expect(connection).to have_received(:execute).with( + a_string_matching( + /CREATE TABLE IF NOT EXISTS "partitions_dynamic"."events_202301".*LIKE "events" INCLUDING ALL/ + ) + ) + expect(connection).to have_received(:execute).with( + a_string_matching( + /ALTER TABLE "events" ATTACH PARTITION "partitions_dynamic"."events_202301".*FOR VALUES FROM/ + ) + ) + end + + it 'creates multiple partitions in a single transaction' do + partitions = [ + Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-01-01', '2023-02-01', partition_name: 'events_202301' + ), + Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-02-01', '2023-03-01', partition_name: 'events_202302' + ) + ] + allow(partitioning_strategy).to receive(:partitions_to_create).and_return(partitions) + + manager = described_class.new(model) + manager.create_partitions + + expect(connection).to have_received(:transaction).once + expect(connection).to have_received(:execute).with(a_string_matching(/events_202301/)).twice + expect(connection).to have_received(:execute).with(a_string_matching(/events_202302/)).twice + end + + it 'does nothing when there are no partitions to create' do + allow(partitioning_strategy).to receive(:partitions_to_create).and_return([]) + + manager = described_class.new(model) + manager.create_partitions + + expect(connection).not_to have_received(:execute).with(a_string_matching(/CREATE TABLE/)) + end + end + + describe '#detach_partitions' do + it 'executes DETACH PARTITION and records metadata in a comment' do + freeze_time do + partition = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-01-01', '2023-02-01', partition_name: 'events_202301' + ) + allow(partitioning_strategy).to receive(:partitions_to_detach).and_return([partition]) + + manager = described_class.new(model) + manager.detach_partitions + + expect(connection).to have_received(:execute).with( + a_string_matching(/ALTER TABLE "events" DETACH PARTITION "partitions_dynamic"."events_202301"/) + ) + expect(connection).to have_received(:execute).with( + a_string_matching(/COMMENT ON TABLE "partitions_dynamic"."events_202301"/) + ) + end + end + + it 'includes detached_at timestamp in the comment' do + freeze_time do + partition = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-01-01', '2023-02-01', partition_name: 'events_202301' + ) + allow(partitioning_strategy).to receive(:partitions_to_detach).and_return([partition]) + + manager = described_class.new(model) + manager.detach_partitions + + expect(connection).to have_received(:execute).with( + a_string_matching(/COMMENT.*#{Regexp.escape(Time.current.iso8601)}/) + ) + end + end + + it 'does nothing when there are no partitions to detach' do + allow(partitioning_strategy).to receive(:partitions_to_detach).and_return([]) + + manager = described_class.new(model) + manager.detach_partitions + + expect(connection).not_to have_received(:execute).with(a_string_matching(/DETACH/)) + end + end + + describe '#drop_partitions' do + it 'executes DROP TABLE for each detached partition' do + detached_partition = double('DetachedPartition', schema: 'partitions_dynamic', + name: 'events_202301', + parent_identifier: 'events') + allow(partitioning_strategy).to receive(:partitions_to_drop).and_return([detached_partition]) + + manager = described_class.new(model) + manager.drop_partitions + + expect(connection).to have_received(:execute).with( + a_string_matching(/DROP TABLE "partitions_dynamic"."events_202301"/) + ) + end + + it 'does nothing when there are no partitions to drop' do + allow(partitioning_strategy).to receive(:partitions_to_drop).and_return([]) + + manager = described_class.new(model) + manager.drop_partitions + + expect(connection).not_to have_received(:execute).with(a_string_matching(/DROP TABLE/)) + end + end + + describe '.sync_all_partitions' do + it 'syncs partitions for each registered model' do + models = [model] + allow(described_class).to receive(:models).and_return(models) + allow(partitioning_strategy).to receive_messages( + partitions_to_create: [], + partitions_to_detach: [], + partitions_to_drop: [] + ) + + described_class.sync_all_partitions + + expect(partitioning_strategy).to have_received(:partitions_to_create) + expect(partitioning_strategy).to have_received(:partitions_to_detach) + expect(partitioning_strategy).to have_received(:partitions_to_drop) + end + end + + describe 'locking' do + it 'acquires an advisory lock before executing partition operations' do + allow(partitioning_strategy).to receive(:partitions_to_create).and_return([]) + + manager = described_class.new(model) + + call_order = [] + allow(connection).to receive(:execute) do |sql| + call_order << sql + end + + manager.create_partitions + + expect(call_order.first).to match(/pg_advisory_xact_lock/) + end + + it 'uses a lock key scoped to the table name' do + allow(partitioning_strategy).to receive(:partitions_to_create).and_return([]) + + other_model = double('OtherModel', table_name: 'other_events') + allow(other_model).to receive(:try).with(:partitioning_strategy).and_return(partitioning_strategy) + allow(other_model).to receive(:partitioning_strategy).and_return(partitioning_strategy) + allow(other_model).to receive(:with_connection).and_yield(connection) + + lock_keys = [] + allow(connection).to receive(:execute) do |sql| + lock_keys << sql.match(/pg_advisory_xact_lock\((\d+)\)/)[1] if sql.match?(/pg_advisory_xact_lock/) + end + + described_class.new(model).create_partitions + described_class.new(other_model).create_partitions + + expect(lock_keys.size).to eq(2) + expect(lock_keys[0]).not_to eq(lock_keys[1]) + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/spec/code0/zero_track/database/partitioning/partitioned_table_spec.rb b/spec/code0/zero_track/database/partitioning/partitioned_table_spec.rb new file mode 100644 index 0000000..e02b9f6 --- /dev/null +++ b/spec/code0/zero_track/database/partitioning/partitioned_table_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/VerifiedDoubles -- AR models require a DB connection for verification. +RSpec.describe Code0::ZeroTrack::Database::Partitioning::PartitionedTable do + let(:test_class) do + Class.new(ActiveRecord::Base) do # rubocop:disable Rails/ApplicationRecord + self.table_name = 'test_partitioned' + include Code0::ZeroTrack::Database::Partitioning::PartitionedTable + end + end + + let(:partitioned_table) { double('PostgresPartitionedTable') } + let(:postgres_partitions_relation) { double('relation') } + + before do + allow(Code0::ZeroTrack::Database::Partitioning::PostgresPartitionedTable) + .to receive(:find_by).and_return(partitioned_table) + allow(partitioned_table).to receive(:postgres_partitions).and_return(postgres_partitions_relation) + allow(postgres_partitions_relation).to receive(:map).and_return([]) + end + + describe '.partition_by' do + it 'configures the model with a daily strategy' do + test_class.partition_by(:created_at, strategy: :daily) + + expect(test_class.partitioning_strategy).to be_a( + Code0::ZeroTrack::Database::Partitioning::Strategy::Daily + ) + expect(test_class.partitioning_strategy.partitioning_column).to eq(:created_at) + end + + it 'configures the model with a monthly strategy' do + test_class.partition_by(:created_at, strategy: :monthly) + + expect(test_class.partitioning_strategy).to be_a( + Code0::ZeroTrack::Database::Partitioning::Strategy::Monthly + ) + end + + it 'passes configuration options to the strategy' do + test_class.partition_by(:created_at, strategy: :daily, retain_for: 90.days, headroom: 14.days) + + strategy = test_class.partitioning_strategy + expect(strategy.retain_for).to eq(90.days) + expect(strategy.headroom).to eq(14.days) + end + + it 'raises ArgumentError for unknown strategy' do + expect do + test_class.partition_by(:created_at, strategy: :yearly) + end.to raise_error(ArgumentError, /Unknown partitioning strategy/) + end + + it 'raises ArgumentError if table is already partitioned' do + test_class.partition_by(:created_at, strategy: :daily) + + expect do + test_class.partition_by(:created_at, strategy: :monthly) + end.to raise_error(ArgumentError, /already partitioned/) + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/spec/code0/zero_track/database/partitioning/strategy/base_spec.rb b/spec/code0/zero_track/database/partitioning/strategy/base_spec.rb new file mode 100644 index 0000000..7a8e659 --- /dev/null +++ b/spec/code0/zero_track/database/partitioning/strategy/base_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/VerifiedDoubles -- AR models require a DB connection for verification. +RSpec.describe Code0::ZeroTrack::Database::Partitioning::Strategy::Base do + let(:model) { double('Model', table_name: 'events') } + + describe '#initialize' do + it 'requires headroom since default_headroom is abstract' do + expect { described_class.new(model, :created_at) }.to raise_error(NotImplementedError) + end + + it 'stores configuration' do + strategy = described_class.new(model, :created_at, headroom: 1.day, retain_for: 90.days, + retain_detached_for: 14.days) + + expect(strategy.model).to eq(model) + expect(strategy.partitioning_column).to eq(:created_at) + expect(strategy.headroom).to eq(1.day) + expect(strategy.retain_for).to eq(90.days) + expect(strategy.retain_detached_for).to eq(14.days) + end + + it 'defaults retain_for to nil and retain_detached_for to 7 days' do + strategy = described_class.new(model, :created_at, headroom: 1.day) + + expect(strategy.retain_for).to be_nil + expect(strategy.retain_detached_for).to eq(7.days) + end + end + + describe '#retention_enabled?' do + it 'returns false when retain_for is not set' do + strategy = described_class.new(model, :created_at, headroom: 1.day) + + expect(strategy.retention_enabled?).to be(false) + end + + it 'returns true when retain_for is set' do + strategy = described_class.new(model, :created_at, headroom: 1.day, retain_for: 90.days) + + expect(strategy.retention_enabled?).to be(true) + end + end + + describe '#partitions_to_drop' do + it 'returns detached partitions older than retain_detached_for' do + strategy = described_class.new(model, :created_at, headroom: 1.day, retain_detached_for: 7.days) + + partitioned_table = double('PostgresPartitionedTable') + detached_partitions_relation = double('relation') + old_partitions = [double('old_partition')] + + allow(Code0::ZeroTrack::Database::Partitioning::PostgresPartitionedTable) + .to receive(:find_by).with(identifier: 'events').and_return(partitioned_table) + allow(partitioned_table).to receive(:postgres_detached_partitions).and_return(detached_partitions_relation) + allow(detached_partitions_relation).to receive(:detached_before).and_return(old_partitions) + + expect(strategy.partitions_to_drop).to eq(old_partitions) + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/spec/code0/zero_track/database/partitioning/strategy/daily_spec.rb b/spec/code0/zero_track/database/partitioning/strategy/daily_spec.rb new file mode 100644 index 0000000..6485227 --- /dev/null +++ b/spec/code0/zero_track/database/partitioning/strategy/daily_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/VerifiedDoubles -- AR models require a DB connection for verification. +RSpec.describe Code0::ZeroTrack::Database::Partitioning::Strategy::Daily do + include ActiveSupport::Testing::TimeHelpers + + let(:model) { double('Model', table_name: 'events') } + + let(:partitioned_table) { double('PostgresPartitionedTable') } + let(:postgres_partitions_relation) { double('relation') } + + before do + allow(Code0::ZeroTrack::Database::Partitioning::PostgresPartitionedTable) + .to receive(:find_by).with(identifier: 'events').and_return(partitioned_table) + allow(partitioned_table).to receive(:postgres_partitions).and_return(postgres_partitions_relation) + allow(postgres_partitions_relation).to receive(:map).and_return([]) + end + + describe '#default_headroom' do + it 'defaults to 30 days' do + strategy = described_class.new(model, :created_at) + + expect(strategy.headroom).to eq(30.days) + end + end + + describe '#partition_name' do + it 'generates name with YYYYMMDD suffix' do + strategy = described_class.new(model, :created_at) + + expect(strategy.partition_name(Date.new(2023, 3, 15))).to eq('events_20230315') + end + + it 'generates zeroed suffix for nil (MINVALUE partition)' do + strategy = described_class.new(model, :created_at) + + expect(strategy.partition_name(nil)).to eq('events_00000000') + end + end + + describe '#desired_partitions' do + context 'with retention enabled' do + it 'starts from the retention boundary and extends through headroom' do + travel_to Date.new(2023, 6, 15) do + strategy = described_class.new(model, :created_at, retain_for: 7.days, headroom: 3.days) + + partitions = strategy.desired_partitions + + expect(partitions.first.from).to eq(Date.new(2023, 6, 8)) + expect(partitions.first.partition_name).to eq('events_20230608') + expect(partitions.last.to).to eq(Date.new(2023, 6, 19)) # current + 1 day + 3 days headroom + expect(partitions).to all(satisfy { |p| p.to - p.from == 1 }) + end + end + end + + context 'without retention' do + it 'starts from today when no partitions exist yet' do + travel_to Date.new(2023, 6, 15) do + strategy = described_class.new(model, :created_at, headroom: 3.days) + + partitions = strategy.desired_partitions + + expect(partitions.first.from).to eq(Date.new(2023, 6, 15)) + expect(partitions.last.to).to eq(Date.new(2023, 6, 19)) + end + end + + it 'starts from the earliest existing partition when partitions exist' do + travel_to Date.new(2023, 6, 15) do + existing = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-06-01', '2023-06-02', partition_name: 'events_20230601' + ) + allow(postgres_partitions_relation).to receive(:map).and_return([existing]) + + strategy = described_class.new(model, :created_at, headroom: 3.days) + + partitions = strategy.desired_partitions + + expect(partitions.first.from).to eq(Date.new(2023, 6, 1)) + end + end + end + + it 'generates one partition per day covering the entire range' do + travel_to Date.new(2023, 6, 15) do + strategy = described_class.new(model, :created_at, retain_for: 2.days, headroom: 2.days) + + partitions = strategy.desired_partitions + + # Verify contiguous: each partition's `to` equals the next partition's `from` + partitions.each_cons(2) do |a, b| + expect(a.to).to eq(b.from) + end + + # Verify each partition spans exactly 1 day + partitions.each do |p| + expect(p.to - p.from).to eq(1) + end + end + end + end + + describe '#current_partitions' do + it 'parses existing partition records into TimePartition objects' do + partition_record = double('record', name: 'events_20230101', + condition: "FOR VALUES FROM ('2023-01-01') TO ('2023-01-02')") + + allow(postgres_partitions_relation).to receive(:map).and_yield(partition_record).and_return( + [Code0::ZeroTrack::Database::Partitioning::TimePartition.from_sql( + model, partition_record.name, partition_record.condition + )] + ) + + strategy = described_class.new(model, :created_at) + partitions = strategy.current_partitions + + expect(partitions.size).to eq(1) + expect(partitions.first.partition_name).to eq('events_20230101') + expect(partitions.first.from).to eq(Date.new(2023, 1, 1)) + expect(partitions.first.to).to eq(Date.new(2023, 1, 2)) + end + end + + describe 'partition syncing logic' do + it 'identifies new partitions to create and old partitions to detach' do + travel_to Date.new(2023, 6, 15) do + # Existing: June 13, 14, 15. Retention: 1 day. Headroom: 2 days. + existing_partitions = (13..15).map do |day| + Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, "2023-06-#{day}", "2023-06-#{day + 1}", partition_name: "events_202306#{day}" + ) + end + allow(postgres_partitions_relation).to receive(:map).and_return(existing_partitions) + + # retain_for 1 day => oldest_active_date = June 14 + # headroom 2 days => max_date = June 15 + 1 + 2 = June 18 + strategy = described_class.new(model, :created_at, retain_for: 1.day, headroom: 2.days) + + # June 13 is before retention boundary (June 14), should be detached + expect(strategy.partitions_to_detach.map(&:partition_name)).to include('events_20230613') + expect(strategy.partitions_to_detach.map(&:partition_name)).not_to include('events_20230614') + + # June 16 and 17 are within headroom but don't exist, should be created + expect(strategy.partitions_to_create.map(&:partition_name)).to include('events_20230616', 'events_20230617') + end + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/spec/code0/zero_track/database/partitioning/strategy/monthly_spec.rb b/spec/code0/zero_track/database/partitioning/strategy/monthly_spec.rb new file mode 100644 index 0000000..e595329 --- /dev/null +++ b/spec/code0/zero_track/database/partitioning/strategy/monthly_spec.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/VerifiedDoubles -- AR models require a DB connection for verification. +RSpec.describe Code0::ZeroTrack::Database::Partitioning::Strategy::Monthly do + include ActiveSupport::Testing::TimeHelpers + + let(:model) { double('Model', table_name: 'events') } + + let(:partitioned_table) { double('PostgresPartitionedTable') } + let(:postgres_partitions_relation) { double('relation') } + + before do + allow(Code0::ZeroTrack::Database::Partitioning::PostgresPartitionedTable) + .to receive(:find_by).with(identifier: 'events').and_return(partitioned_table) + allow(partitioned_table).to receive(:postgres_partitions).and_return(postgres_partitions_relation) + allow(postgres_partitions_relation).to receive(:map).and_return([]) + end + + describe '#default_headroom' do + it 'defaults to 6 months' do + strategy = described_class.new(model, :created_at) + + expect(strategy.headroom).to eq(6.months) + end + end + + describe '#partition_name' do + it 'generates name with YYYYMM suffix' do + strategy = described_class.new(model, :created_at) + + expect(strategy.partition_name(Date.new(2023, 3, 15))).to eq('events_202303') + end + + it 'generates zeroed suffix for nil (MINVALUE partition)' do + strategy = described_class.new(model, :created_at) + + expect(strategy.partition_name(nil)).to eq('events_000000') + end + end + + describe '#desired_partitions' do + context 'with retention enabled' do + it 'starts from the retention boundary at beginning of month' do + travel_to Date.new(2023, 6, 15) do + strategy = described_class.new(model, :created_at, retain_for: 3.months, headroom: 2.months) + + partitions = strategy.desired_partitions + + # 3 months ago from June 15 = March 15, beginning_of_month = March 1 + expect(partitions.first.from).to eq(Date.new(2023, 3, 1)) + expect(partitions.first.partition_name).to eq('events_202303') + # end_of_month of June = June 30, + 2 months = August 30 + expect(partitions.last.to).to be >= Date.new(2023, 8, 1) + end + end + end + + context 'without retention' do + it 'starts from beginning of current month when no partitions exist' do + travel_to Date.new(2023, 6, 15) do + strategy = described_class.new(model, :created_at, headroom: 2.months) + + partitions = strategy.desired_partitions + + expect(partitions.first.from).to eq(Date.new(2023, 6, 1)) + end + end + + it 'starts from the earliest existing partition month when partitions exist' do + travel_to Date.new(2023, 6, 15) do + existing = Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, '2023-03-01', '2023-04-01', partition_name: 'events_202303' + ) + allow(postgres_partitions_relation).to receive(:map).and_return([existing]) + + strategy = described_class.new(model, :created_at, headroom: 2.months) + + partitions = strategy.desired_partitions + + expect(partitions.first.from).to eq(Date.new(2023, 3, 1)) + end + end + end + + it 'generates contiguous monthly partitions' do + travel_to Date.new(2023, 6, 15) do + strategy = described_class.new(model, :created_at, retain_for: 2.months, headroom: 2.months) + + partitions = strategy.desired_partitions + + # Verify contiguous: each partition's `to` equals the next partition's `from` + partitions.each_cons(2) do |a, b| + expect(a.to).to eq(b.from) + end + + # Verify each partition starts on the 1st + partitions.each do |p| + expect(p.from.day).to eq(1) + expect(p.to.day).to eq(1) + end + end + end + end + + describe '#current_partitions' do + it 'parses existing partition records into TimePartition objects' do + partition_record = double('record', name: 'events_202301', + condition: "FOR VALUES FROM ('2023-01-01') TO ('2023-02-01')") + + allow(postgres_partitions_relation).to receive(:map).and_yield(partition_record).and_return( + [Code0::ZeroTrack::Database::Partitioning::TimePartition.from_sql( + model, partition_record.name, partition_record.condition + )] + ) + + strategy = described_class.new(model, :created_at) + partitions = strategy.current_partitions + + expect(partitions.size).to eq(1) + expect(partitions.first.partition_name).to eq('events_202301') + expect(partitions.first.from).to eq(Date.new(2023, 1, 1)) + expect(partitions.first.to).to eq(Date.new(2023, 2, 1)) + end + end + + describe 'partition syncing logic' do + it 'identifies new partitions to create and old partitions to detach' do + travel_to Date.new(2023, 6, 15) do + # Existing: March, April, May, June + existing_partitions = (3..6).map do |month| + from = Date.new(2023, month, 1) + to = from.next_month + Code0::ZeroTrack::Database::Partitioning::TimePartition.new( + model, from, to, partition_name: "events_2023#{format('%02d', month)}" + ) + end + allow(postgres_partitions_relation).to receive(:map).and_return(existing_partitions) + + # retain_for: 2 months (from June 15, 2 months ago = April 15, beginning_of_month = April 1) + # headroom: 2 months (end_of_month June = June 30, + 2 months = August 30) + strategy = described_class.new(model, :created_at, retain_for: 2.months, headroom: 2.months) + + # March is before retention boundary (April 1), should be detached + expect(strategy.partitions_to_detach.map(&:partition_name)).to include('events_202303') + expect(strategy.partitions_to_detach.map(&:partition_name)).not_to include('events_202304') + + # July and August don't exist yet but are within headroom + expect(strategy.partitions_to_create.map(&:partition_name)).to include('events_202307', 'events_202308') + end + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/spec/code0/zero_track/database/partitioning/time_partition_spec.rb b/spec/code0/zero_track/database/partitioning/time_partition_spec.rb new file mode 100644 index 0000000..504b8c6 --- /dev/null +++ b/spec/code0/zero_track/database/partitioning/time_partition_spec.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop:disable RSpec/VerifiedDoubles -- AR models require a DB connection for verification. +RSpec.describe Code0::ZeroTrack::Database::Partitioning::TimePartition do + let(:model) do + double('Model', table_name: 'events') + end + + let(:connection) do + connection = double('Connection') + allow(connection).to receive(:quote_table_name) { |name| "\"#{name}\"" } + allow(connection).to receive(:quote) { |value| "'#{value}'" } + connection + end + + before do + allow(Rails.application.config.zero_track.db_partitioning).to receive(:dynamic_partition_schema) + .and_return('partitions_dynamic') + end + + describe '.from_sql' do + it 'parses a standard range partition definition' do + partition = described_class.from_sql(model, 'events_202301', "FOR VALUES FROM ('2023-01-01') TO ('2023-02-01')") + + expect(partition.model).to eq(model) + expect(partition.partition_name).to eq('events_202301') + expect(partition.from).to eq(Date.parse('2023-01-01')) + expect(partition.to).to eq(Date.parse('2023-02-01')) + end + + it 'parses a partition with MINVALUE lower bound' do + partition = described_class.from_sql(model, 'events_initial', "FOR VALUES FROM (MINVALUE) TO ('2023-01-01')") + + expect(partition.from).to be_nil + expect(partition.to).to eq(Date.parse('2023-01-01')) + end + + it 'raises ArgumentError for unknown definition format' do + expect do + described_class.from_sql(model, 'events_bad', 'SOMETHING UNEXPECTED') + end.to raise_error(ArgumentError, /Unknown partition definition/) + end + + it 'raises NotImplementedError for MAXVALUE upper bound' do + expect do + described_class.from_sql(model, 'events_max', "FOR VALUES FROM ('2023-01-01') TO (MAXVALUE)") + end.to raise_error(NotImplementedError) + end + end + + describe '#initialize' do + it 'parses string dates' do + partition = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + expect(partition.from).to eq(Date.new(2023, 1, 1)) + expect(partition.to).to eq(Date.new(2023, 2, 1)) + end + + it 'accepts Date objects' do + from = Date.new(2023, 1, 1) + to = Date.new(2023, 2, 1) + partition = described_class.new(model, from, to, partition_name: 'events_202301') + + expect(partition.from).to eq(from) + expect(partition.to).to eq(to) + end + + it 'allows nil from for MINVALUE partitions' do + partition = described_class.new(model, nil, '2023-02-01', partition_name: 'events_initial') + + expect(partition.from).to be_nil + end + end + + describe 'equality and comparison' do + it 'treats partitions with identical attributes as equal' do + p1 = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + p2 = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + expect(p1).to eq(p2) + end + + it 'treats partitions with any differing attribute as not equal' do + base = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + different_name = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_other') + different_from = described_class.new(model, '2023-01-15', '2023-02-01', partition_name: 'events_202301') + different_to = described_class.new(model, '2023-01-01', '2023-03-01', partition_name: 'events_202301') + + expect(base).not_to eq(different_name) + expect(base).not_to eq(different_from) + expect(base).not_to eq(different_to) + end + + it 'can be used in sets for deduplication' do + p1 = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + p2 = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + p3 = described_class.new(model, '2023-02-01', '2023-03-01', partition_name: 'events_202302') + + set = Set.new([p1, p2, p3]) + + expect(set.size).to eq(2) + end + + it 'sorts partitions by name within the same model' do + p1 = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + p2 = described_class.new(model, '2023-02-01', '2023-03-01', partition_name: 'events_202302') + p3 = described_class.new(model, '2023-03-01', '2023-04-01', partition_name: 'events_202303') + + expect([p3, p1, p2].sort).to eq([p1, p2, p3]) + end + + it 'cannot compare partitions across different models' do + other_model = double('OtherModel', table_name: 'other') + p1 = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + p2 = described_class.new(other_model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + expect(p1 <=> p2).to be_nil + end + end + + describe 'set arithmetic for partition syncing' do + it 'computes partitions to create via array subtraction' do + existing = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + desired_new = described_class.new(model, '2023-02-01', '2023-03-01', partition_name: 'events_202302') + + desired = [existing, desired_new] + current = [existing] + + to_create = desired - current + + expect(to_create).to eq([desired_new]) + end + + it 'computes partitions to detach via array subtraction' do + kept = described_class.new(model, '2023-02-01', '2023-03-01', partition_name: 'events_202302') + old = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + desired = [kept] + current = [kept, old] + + to_detach = current - desired + + expect(to_detach).to eq([old]) + end + end + + describe '#to_create_sql' do + it 'generates CREATE TABLE IF NOT EXISTS in the dynamic schema' do + partition = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + sql = partition.to_create_sql(connection) + + expect(sql).to eq( + 'CREATE TABLE IF NOT EXISTS "partitions_dynamic"."events_202301" (LIKE "events" INCLUDING ALL)' + ) + end + end + + describe '#to_attach_sql' do + it 'generates ATTACH PARTITION with date range bounds' do + partition = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + sql = partition.to_attach_sql(connection) + + expect(sql).to eq( + 'ALTER TABLE "events" ATTACH PARTITION "partitions_dynamic"."events_202301" ' \ + "FOR VALUES FROM ('2023-01-01') TO ('2023-02-01')" + ) + end + + it 'uses MINVALUE when from is nil' do + partition = described_class.new(model, nil, '2023-01-01', partition_name: 'events_initial') + + sql = partition.to_attach_sql(connection) + + expect(sql).to include('FOR VALUES FROM (MINVALUE) TO') + end + end + + describe '#to_detach_sql' do + it 'generates DETACH PARTITION' do + partition = described_class.new(model, '2023-01-01', '2023-02-01', partition_name: 'events_202301') + + sql = partition.to_detach_sql(connection) + + expect(sql).to eq( + 'ALTER TABLE "events" DETACH PARTITION "partitions_dynamic"."events_202301"' + ) + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/spec/code0/zero_track/database/schema_cleaner_spec.rb b/spec/code0/zero_track/database/schema_cleaner_spec.rb new file mode 100644 index 0000000..ee5a94d --- /dev/null +++ b/spec/code0/zero_track/database/schema_cleaner_spec.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Code0::ZeroTrack::Database::SchemaCleaner do + before do + allow(Rails.application.config.zero_track.db_partitioning).to receive(:dynamic_partition_schema) + .and_return('partitions_dynamic') + end + + describe '#clean' do + subject(:clean) do + output = StringIO.new + described_class.new(input).clean(output) + output.string + end + + context 'when removing general noise' do + let(:input) do + <<~SQL + SET statement_timeout = 0; + SET lock_timeout = 0; + + SELECT pg_catalog.set_config('search_path', '', false); + + -- This is a comment + + COMMENT ON EXTENSION "plpgsql" IS 'PL/pgSQL procedural language'; + + CREATE TABLE users ( + id bigint NOT NULL, + name text NOT NULL + ); + SQL + end + + it 'removes SET statements' do + expect(clean).not_to include('SET ') + end + + it 'removes SELECT pg_catalog' do + expect(clean).not_to include('pg_catalog') + end + + it 'removes comments' do + expect(clean).not_to include('-- This is a comment') + end + + it 'removes COMMENT ON EXTENSION' do + expect(clean).not_to include('COMMENT ON EXTENSION') + end + + it 'preserves table definitions' do + expect(clean).to include('CREATE TABLE users') + end + end + + context 'when removing public schema qualifications' do + let(:input) do + <<~SQL # rubocop:disable Rails/SquishedSQLHeredocs -- this should match how a structure.sql is generated + CREATE TABLE public.users ( + id bigint NOT NULL + ); + + CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA public; + SQL + end + + it 'removes public. prefix from identifiers' do + expect(clean).to include('CREATE TABLE users') + expect(clean).not_to include('public.users') + end + + it 'removes WITH SCHEMA public from extensions' do + expect(clean).to include('CREATE EXTENSION IF NOT EXISTS plpgsql;') + expect(clean).not_to include('WITH SCHEMA public') + end + end + + context 'when removing dynamic partition objects' do + let(:input) do + <<~SQL # rubocop:disable Rails/SquishedSQLHeredocs -- this should match how a structure.sql is generated + CREATE TABLE p_audit_events ( + id bigint NOT NULL, + author_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL + ) + PARTITION BY RANGE (created_at); + + CREATE TABLE partitions_dynamic.p_audit_events_202607 ( + id bigint NOT NULL, + author_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL + ); + + CREATE TABLE partitions_dynamic.p_audit_events_202608 ( + id bigint NOT NULL, + author_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL + ); + + CREATE TABLE users ( + id bigint NOT NULL, + name text NOT NULL + ); + + ALTER TABLE ONLY p_audit_events ATTACH PARTITION partitions_dynamic.p_audit_events_202607 FOR VALUES FROM ('2026-07-01 00:00:00+00') TO ('2026-08-01 00:00:00+00'); + + ALTER TABLE ONLY p_audit_events ATTACH PARTITION partitions_dynamic.p_audit_events_202608 FOR VALUES FROM ('2026-08-01 00:00:00+00') TO ('2026-09-01 00:00:00+00'); + + ALTER TABLE ONLY p_audit_events + ADD CONSTRAINT p_audit_events_pkey PRIMARY KEY (id, created_at); + + ALTER TABLE ONLY partitions_dynamic.p_audit_events_202607 + ADD CONSTRAINT p_audit_events_202607_pkey PRIMARY KEY (id, created_at); + + ALTER TABLE ONLY partitions_dynamic.p_audit_events_202608 + ADD CONSTRAINT p_audit_events_202608_pkey PRIMARY KEY (id, created_at); + + ALTER TABLE ONLY users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + CREATE INDEX index_p_audit_events_on_author_id ON ONLY p_audit_events USING btree (author_id); + + CREATE INDEX p_audit_events_202607_author_id_idx ON partitions_dynamic.p_audit_events_202607 USING btree (author_id); + + CREATE INDEX p_audit_events_202608_author_id_idx ON partitions_dynamic.p_audit_events_202608 USING btree (author_id); + + CREATE UNIQUE INDEX index_users_on_name ON users USING btree (name); + + ALTER INDEX index_p_audit_events_on_author_id ATTACH PARTITION partitions_dynamic.p_audit_events_202607_author_id_idx; + + ALTER INDEX p_audit_events_pkey ATTACH PARTITION partitions_dynamic.p_audit_events_202607_pkey; + + ALTER INDEX index_p_audit_events_on_author_id ATTACH PARTITION partitions_dynamic.p_audit_events_202608_author_id_idx; + + ALTER INDEX p_audit_events_pkey ATTACH PARTITION partitions_dynamic.p_audit_events_202608_pkey; + + ALTER TABLE p_audit_events + ADD CONSTRAINT fk_rails_9c5a4c4493 FOREIGN KEY (author_id) REFERENCES users(id); + SQL + end + + it 'removes CREATE TABLE for dynamic partitions' do + expect(clean).not_to include('partitions_dynamic.p_audit_events_202607') + expect(clean).not_to include('partitions_dynamic.p_audit_events_202608') + end + + it 'preserves the parent partitioned table' do + expect(clean).to include('CREATE TABLE p_audit_events') + expect(clean).to include('PARTITION BY RANGE (created_at)') + end + + it 'preserves unrelated tables' do + expect(clean).to include('CREATE TABLE users') + end + + it 'removes ATTACH PARTITION statements for dynamic partitions' do + expect(clean).not_to include('ATTACH PARTITION partitions_dynamic.') + end + + it 'removes ALTER TABLE ONLY for dynamic partition constraints' do + expect(clean).not_to include('p_audit_events_202607_pkey') + expect(clean).not_to include('p_audit_events_202608_pkey') + end + + it 'preserves constraints on the parent table' do + expect(clean).to include('p_audit_events_pkey PRIMARY KEY (id, created_at)') + end + + it 'preserves constraints on unrelated tables' do + expect(clean).to include('users_pkey PRIMARY KEY (id)') + end + + it 'removes CREATE INDEX on dynamic partitions' do + expect(clean).not_to include('p_audit_events_202607_author_id_idx') + expect(clean).not_to include('p_audit_events_202608_author_id_idx') + end + + it 'preserves indexes on the parent table' do + expect(clean).to include('CREATE INDEX index_p_audit_events_on_author_id ON ONLY p_audit_events') + end + + it 'preserves indexes on unrelated tables' do + expect(clean).to include('CREATE UNIQUE INDEX index_users_on_name ON users') + end + + it 'removes ALTER INDEX ... ATTACH PARTITION for dynamic partitions' do + expect(clean).not_to include('ALTER INDEX index_p_audit_events_on_author_id ATTACH PARTITION') + expect(clean).not_to include('ALTER INDEX p_audit_events_pkey ATTACH PARTITION') + end + + it 'preserves foreign keys on the parent table' do + expect(clean).to include('fk_rails_9c5a4c4493 FOREIGN KEY (author_id) REFERENCES users(id)') + end + end + + context 'with a custom dynamic partition schema name' do + before do + allow(Rails.application.config.zero_track.db_partitioning).to receive(:dynamic_partition_schema) + .and_return('custom_partitions') + end + + let(:input) do + <<~SQL # rubocop:disable Rails/SquishedSQLHeredocs -- this should match how a structure.sql is generated + CREATE TABLE custom_partitions.events_202607 ( + id bigint NOT NULL + ); + + CREATE TABLE partitions_dynamic.unrelated_202607 ( + id bigint NOT NULL + ); + SQL + end + + it 'removes partitions from the configured schema' do + expect(clean).not_to include('custom_partitions.events_202607') + end + + it 'does not remove partitions from other schemas' do + expect(clean).to include('partitions_dynamic.unrelated_202607') + end + end + end +end