From fca3662473d175a6948d7968483cf44e0e0a9da7 Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Wed, 29 Mar 2023 10:32:05 -0700 Subject: [PATCH] Add history size bytes to SQL visibility (#4090) --- common/persistence/sql/sqlplugin/visibility.go | 1 + .../visibility/store/sql/visibility_store.go | 4 ++++ schema/mysql/v8/version.go | 2 +- schema/mysql/v8/visibility/schema.sql | 16 +++++++++------- .../versioned/v1.3/add_history_size_bytes.sql | 2 ++ .../v8/visibility/versioned/v1.3/manifest.json | 8 ++++++++ schema/postgresql/v12/version.go | 2 +- schema/postgresql/v12/visibility/schema.sql | 16 +++++++++------- .../versioned/v1.3/add_history_size_bytes.sql | 2 ++ .../v12/visibility/versioned/v1.3/manifest.json | 8 ++++++++ schema/sqlite/v3/visibility/schema.sql | 16 +++++++++------- 11 files changed, 54 insertions(+), 23 deletions(-) create mode 100644 schema/mysql/v8/visibility/versioned/v1.3/add_history_size_bytes.sql create mode 100644 schema/mysql/v8/visibility/versioned/v1.3/manifest.json create mode 100644 schema/postgresql/v12/visibility/versioned/v1.3/add_history_size_bytes.sql create mode 100644 schema/postgresql/v12/visibility/versioned/v1.3/manifest.json diff --git a/common/persistence/sql/sqlplugin/visibility.go b/common/persistence/sql/sqlplugin/visibility.go index 0bf8076a213..fbb4022bfb8 100644 --- a/common/persistence/sql/sqlplugin/visibility.go +++ b/common/persistence/sql/sqlplugin/visibility.go @@ -56,6 +56,7 @@ type ( Status int32 CloseTime *time.Time HistoryLength *int64 + HistorySizeBytes *int64 Memo []byte Encoding string TaskQueue string diff --git a/common/persistence/visibility/store/sql/visibility_store.go b/common/persistence/visibility/store/sql/visibility_store.go index 7edc3c59f3f..aba4672aace 100644 --- a/common/persistence/visibility/store/sql/visibility_store.go +++ b/common/persistence/visibility/store/sql/visibility_store.go @@ -135,6 +135,7 @@ func (s *VisibilityStore) RecordWorkflowExecutionClosed( CloseTime: &request.CloseTime, Status: int32(request.Status), HistoryLength: &request.HistoryLength, + HistorySizeBytes: &request.HistorySizeBytes, Memo: request.Memo.Data, Encoding: request.Memo.EncodingType.String(), TaskQueue: request.TaskQueue, @@ -562,6 +563,9 @@ func (s *VisibilityStore) rowToInfo( if row.HistoryLength != nil { info.HistoryLength = *row.HistoryLength } + if row.HistorySizeBytes != nil { + info.HistorySizeBytes = *row.HistorySizeBytes + } return info, nil } diff --git a/schema/mysql/v8/version.go b/schema/mysql/v8/version.go index a7f1301985e..7c053bc0f6f 100644 --- a/schema/mysql/v8/version.go +++ b/schema/mysql/v8/version.go @@ -30,4 +30,4 @@ package v8 const Version = "1.9" // VisibilityVersion is the MySQL visibility database release version -const VisibilityVersion = "1.2" +const VisibilityVersion = "1.3" diff --git a/schema/mysql/v8/visibility/schema.sql b/schema/mysql/v8/visibility/schema.sql index 08540662461..65fc576ff4d 100644 --- a/schema/mysql/v8/visibility/schema.sql +++ b/schema/mysql/v8/visibility/schema.sql @@ -8,6 +8,7 @@ CREATE TABLE executions_visibility ( status INT NOT NULL, -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT} close_time DATETIME(6) NULL, history_length BIGINT NULL, + history_size_bytes BIGINT NULL, memo BLOB NULL, encoding VARCHAR(64) NOT NULL, task_queue VARCHAR(255) NOT NULL DEFAULT '', @@ -40,13 +41,14 @@ CREATE TABLE executions_visibility ( PRIMARY KEY (namespace_id, run_id) ); -CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); -- Indexes for the predefined search attributes CREATE INDEX by_temporal_change_version ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); diff --git a/schema/mysql/v8/visibility/versioned/v1.3/add_history_size_bytes.sql b/schema/mysql/v8/visibility/versioned/v1.3/add_history_size_bytes.sql new file mode 100644 index 00000000000..4a97b64394d --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.3/add_history_size_bytes.sql @@ -0,0 +1,2 @@ +ALTER TABLE executions_visibility ADD COLUMN history_size_bytes BIGINT NULL; +CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); diff --git a/schema/mysql/v8/visibility/versioned/v1.3/manifest.json b/schema/mysql/v8/visibility/versioned/v1.3/manifest.json new file mode 100644 index 00000000000..7a40fe1ffa2 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.3/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.3", + "MinCompatibleVersion": "0.1", + "Description": "add history size bytes", + "SchemaUpdateCqlFiles": [ + "add_history_size_bytes.sql" + ] +} diff --git a/schema/postgresql/v12/version.go b/schema/postgresql/v12/version.go index a350f7fc092..96aa84baae7 100644 --- a/schema/postgresql/v12/version.go +++ b/schema/postgresql/v12/version.go @@ -32,4 +32,4 @@ const Version = "1.9" // VisibilityVersion is the Postgres visibility database release version // Temporal supports both MySQL and Postgres officially, so upgrade should be performed for both MySQL and Postgres -const VisibilityVersion = "1.2" +const VisibilityVersion = "1.3" diff --git a/schema/postgresql/v12/visibility/schema.sql b/schema/postgresql/v12/visibility/schema.sql index d96cfb55a4c..5805e5d85db 100644 --- a/schema/postgresql/v12/visibility/schema.sql +++ b/schema/postgresql/v12/visibility/schema.sql @@ -17,6 +17,7 @@ CREATE TABLE executions_visibility ( status INTEGER NOT NULL, -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT} close_time TIMESTAMP NULL, history_length BIGINT NULL, + history_size_bytes BIGINT NULL, memo BYTEA NULL, encoding VARCHAR(64) NOT NULL, task_queue VARCHAR(255) NOT NULL DEFAULT '', @@ -69,13 +70,14 @@ CREATE TABLE executions_visibility ( PRIMARY KEY (namespace_id, run_id) ); -CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -- Indexes for the predefined search attributes CREATE INDEX by_temporal_change_version ON executions_visibility USING GIN (namespace_id, TemporalChangeVersion jsonb_path_ops); diff --git a/schema/postgresql/v12/visibility/versioned/v1.3/add_history_size_bytes.sql b/schema/postgresql/v12/visibility/versioned/v1.3/add_history_size_bytes.sql new file mode 100644 index 00000000000..3aaf6dab28b --- /dev/null +++ b/schema/postgresql/v12/visibility/versioned/v1.3/add_history_size_bytes.sql @@ -0,0 +1,2 @@ +ALTER TABLE executions_visibility ADD COLUMN history_size_bytes BIGINT NULL; +CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); diff --git a/schema/postgresql/v12/visibility/versioned/v1.3/manifest.json b/schema/postgresql/v12/visibility/versioned/v1.3/manifest.json new file mode 100644 index 00000000000..7a40fe1ffa2 --- /dev/null +++ b/schema/postgresql/v12/visibility/versioned/v1.3/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.3", + "MinCompatibleVersion": "0.1", + "Description": "add history size bytes", + "SchemaUpdateCqlFiles": [ + "add_history_size_bytes.sql" + ] +} diff --git a/schema/sqlite/v3/visibility/schema.sql b/schema/sqlite/v3/visibility/schema.sql index c17ce8081aa..ccc76a19903 100644 --- a/schema/sqlite/v3/visibility/schema.sql +++ b/schema/sqlite/v3/visibility/schema.sql @@ -8,6 +8,7 @@ CREATE TABLE executions_visibility ( status INT NOT NULL, -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT} close_time TIMESTAMP NULL, history_length BIGINT NULL, + history_size_bytes BIGINT NULL, memo BLOB NULL, encoding VARCHAR(64) NOT NULL, task_queue VARCHAR(255) NOT NULL DEFAULT '', @@ -55,13 +56,14 @@ CREATE TABLE executions_visibility ( PRIMARY KEY (namespace_id, run_id) ); -CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX default_idx ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_execution_time ON executions_visibility (namespace_id, execution_time, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_id ON executions_visibility (namespace_id, workflow_id, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_workflow_type ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_status ON executions_visibility (namespace_id, status, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_history_length ON executions_visibility (namespace_id, history_length, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); +CREATE INDEX by_task_queue ON executions_visibility (namespace_id, task_queue, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id); -- Indexes for the predefined search attributes CREATE INDEX by_batcher_user ON executions_visibility (namespace_id, BatcherUser, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);