Add history size bytes to SQL visibility (#4090)

temporalio · Mar 29, 2023 · fca3662 · fca3662
1 parent f4c1888
commit fca3662
Show file tree

Hide file tree

Showing 11 changed files with 54 additions and 23 deletions.
diff --git a/common/persistence/sql/sqlplugin/visibility.go b/common/persistence/sql/sqlplugin/visibility.go
@@ -56,6 +56,7 @@ type (
 		Status           int32
 		CloseTime        *time.Time
 		HistoryLength    *int64
+		HistorySizeBytes *int64
 		Memo             []byte
 		Encoding         string
 		TaskQueue        string

diff --git a/common/persistence/visibility/store/sql/visibility_store.go b/common/persistence/visibility/store/sql/visibility_store.go
@@ -135,6 +135,7 @@ func (s *VisibilityStore) RecordWorkflowExecutionClosed(
 		CloseTime:        &request.CloseTime,
 		Status:           int32(request.Status),
 		HistoryLength:    &request.HistoryLength,
+		HistorySizeBytes: &request.HistorySizeBytes,
 		Memo:             request.Memo.Data,
 		Encoding:         request.Memo.EncodingType.String(),
 		TaskQueue:        request.TaskQueue,
@@ -562,6 +563,9 @@ func (s *VisibilityStore) rowToInfo(
 	if row.HistoryLength != nil {
 		info.HistoryLength = *row.HistoryLength
 	}
+	if row.HistorySizeBytes != nil {
+		info.HistorySizeBytes = *row.HistorySizeBytes
+	}
 	return info, nil
 }
 

diff --git a/schema/mysql/v8/version.go b/schema/mysql/v8/version.go
@@ -30,4 +30,4 @@ package v8
 const Version = "1.9"
 
 // VisibilityVersion is the MySQL visibility database release version
-const VisibilityVersion = "1.2"
+const VisibilityVersion = "1.3"
diff --git a/schema/mysql/v8/visibility/schema.sql b/schema/mysql/v8/visibility/schema.sql
@@ -8,6 +8,7 @@ CREATE TABLE executions_visibility (
   status              INT           NOT NULL,  -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT}
   close_time          DATETIME(6)   NULL,
   history_length      BIGINT        NULL,
+  history_size_bytes  BIGINT        NULL,
   memo                BLOB          NULL,
   encoding            VARCHAR(64)   NOT NULL,
   task_queue          VARCHAR(255)  NOT NULL DEFAULT '',
@@ -40,13 +41,14 @@ CREATE TABLE executions_visibility (
   PRIMARY KEY (namespace_id, run_id)
 );
 
-CREATE INDEX default_idx        ON executions_visibility (namespace_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
-CREATE INDEX by_execution_time  ON executions_visibility (namespace_id, execution_time,     (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
-CREATE INDEX by_workflow_id     ON executions_visibility (namespace_id, workflow_id,        (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
-CREATE INDEX by_workflow_type   ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
-CREATE INDEX by_status          ON executions_visibility (namespace_id, status,             (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
-CREATE INDEX by_history_length  ON executions_visibility (namespace_id, history_length,     (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
-CREATE INDEX by_task_queue      ON executions_visibility (namespace_id, task_queue,         (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX default_idx            ON executions_visibility (namespace_id, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_execution_time      ON executions_visibility (namespace_id, execution_time,     (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_workflow_id         ON executions_visibility (namespace_id, workflow_id,        (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_workflow_type       ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_status              ON executions_visibility (namespace_id, status,             (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_history_length      ON executions_visibility (namespace_id, history_length,     (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_history_size_bytes  ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
+CREATE INDEX by_task_queue          ON executions_visibility (namespace_id, task_queue,         (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
 
 -- Indexes for the predefined search attributes
 CREATE INDEX by_temporal_change_version       ON executions_visibility (namespace_id, (CAST(TemporalChangeVersion AS CHAR(255) ARRAY)), (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);

diff --git a/schema/mysql/v8/visibility/versioned/v1.3/add_history_size_bytes.sql b/schema/mysql/v8/visibility/versioned/v1.3/add_history_size_bytes.sql
@@ -0,0 +1,2 @@
+ALTER TABLE executions_visibility ADD COLUMN history_size_bytes BIGINT NULL;
+CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id);
diff --git a/schema/mysql/v8/visibility/versioned/v1.3/manifest.json b/schema/mysql/v8/visibility/versioned/v1.3/manifest.json
@@ -0,0 +1,8 @@
+{
+  "CurrVersion": "1.3",
+  "MinCompatibleVersion": "0.1",
+  "Description": "add history size bytes",
+  "SchemaUpdateCqlFiles": [
+    "add_history_size_bytes.sql"
+  ]
+}
diff --git a/schema/postgresql/v12/version.go b/schema/postgresql/v12/version.go
@@ -32,4 +32,4 @@ const Version = "1.9"
 
 // VisibilityVersion is the Postgres visibility database release version
 // Temporal supports both MySQL and Postgres officially, so upgrade should be performed for both MySQL and Postgres
-const VisibilityVersion = "1.2"
+const VisibilityVersion = "1.3"
diff --git a/schema/postgresql/v12/visibility/schema.sql b/schema/postgresql/v12/visibility/schema.sql
@@ -17,6 +17,7 @@ CREATE TABLE executions_visibility (
   status              INTEGER       NOT NULL,  -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT}
   close_time          TIMESTAMP     NULL,
   history_length      BIGINT        NULL,
+  history_size_bytes  BIGINT        NULL,
   memo                BYTEA         NULL,
   encoding            VARCHAR(64)   NOT NULL,
   task_queue          VARCHAR(255)  NOT NULL DEFAULT '',
@@ -69,13 +70,14 @@ CREATE TABLE executions_visibility (
   PRIMARY KEY  (namespace_id, run_id)
 );
 
-CREATE INDEX default_idx        ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
-CREATE INDEX by_execution_time  ON executions_visibility (namespace_id, execution_time,     (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
-CREATE INDEX by_workflow_id     ON executions_visibility (namespace_id, workflow_id,        (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
-CREATE INDEX by_workflow_type   ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
-CREATE INDEX by_status          ON executions_visibility (namespace_id, status,             (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
-CREATE INDEX by_history_length  ON executions_visibility (namespace_id, history_length,     (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
-CREATE INDEX by_task_queue      ON executions_visibility (namespace_id, task_queue,         (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX default_idx            ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_execution_time      ON executions_visibility (namespace_id, execution_time,     (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_workflow_id         ON executions_visibility (namespace_id, workflow_id,        (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_workflow_type       ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_status              ON executions_visibility (namespace_id, status,             (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_history_length      ON executions_visibility (namespace_id, history_length,     (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_history_size_bytes  ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
+CREATE INDEX by_task_queue          ON executions_visibility (namespace_id, task_queue,         (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
 
 -- Indexes for the predefined search attributes
 CREATE INDEX by_temporal_change_version       ON executions_visibility USING GIN (namespace_id, TemporalChangeVersion jsonb_path_ops);

diff --git a/schema/postgresql/v12/visibility/versioned/v1.3/add_history_size_bytes.sql b/schema/postgresql/v12/visibility/versioned/v1.3/add_history_size_bytes.sql
@@ -0,0 +1,2 @@
+ALTER TABLE executions_visibility ADD COLUMN history_size_bytes BIGINT NULL;
+CREATE INDEX by_history_size_bytes ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id);
diff --git a/schema/postgresql/v12/visibility/versioned/v1.3/manifest.json b/schema/postgresql/v12/visibility/versioned/v1.3/manifest.json
@@ -0,0 +1,8 @@
+{
+  "CurrVersion": "1.3",
+  "MinCompatibleVersion": "0.1",
+  "Description": "add history size bytes",
+  "SchemaUpdateCqlFiles": [
+    "add_history_size_bytes.sql"
+  ]
+}
diff --git a/schema/sqlite/v3/visibility/schema.sql b/schema/sqlite/v3/visibility/schema.sql
@@ -8,6 +8,7 @@ CREATE TABLE executions_visibility (
   status              INT           NOT NULL,  -- enum WorkflowExecutionStatus {RUNNING, COMPLETED, FAILED, CANCELED, TERMINATED, CONTINUED_AS_NEW, TIMED_OUT}
   close_time          TIMESTAMP     NULL,
   history_length      BIGINT        NULL,
+  history_size_bytes  BIGINT        NULL,
   memo                BLOB          NULL,
   encoding            VARCHAR(64)   NOT NULL,
   task_queue          VARCHAR(255)  NOT NULL DEFAULT '',
@@ -55,13 +56,14 @@ CREATE TABLE executions_visibility (
   PRIMARY KEY (namespace_id, run_id)
 );
 
-CREATE INDEX default_idx        ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
-CREATE INDEX by_execution_time  ON executions_visibility (namespace_id, execution_time,     (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
-CREATE INDEX by_workflow_id     ON executions_visibility (namespace_id, workflow_id,        (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
-CREATE INDEX by_workflow_type   ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
-CREATE INDEX by_status          ON executions_visibility (namespace_id, status,             (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
-CREATE INDEX by_history_length  ON executions_visibility (namespace_id, history_length,     (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
-CREATE INDEX by_task_queue      ON executions_visibility (namespace_id, task_queue,         (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX default_idx            ON executions_visibility (namespace_id, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_execution_time      ON executions_visibility (namespace_id, execution_time,     (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_workflow_id         ON executions_visibility (namespace_id, workflow_id,        (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_workflow_type       ON executions_visibility (namespace_id, workflow_type_name, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_status              ON executions_visibility (namespace_id, status,             (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_history_length      ON executions_visibility (namespace_id, history_length,     (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_history_size_bytes  ON executions_visibility (namespace_id, history_size_bytes, (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
+CREATE INDEX by_task_queue          ON executions_visibility (namespace_id, task_queue,         (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);
 
 -- Indexes for the predefined search attributes
 CREATE INDEX by_batcher_user                  ON executions_visibility (namespace_id, BatcherUser,                (COALESCE(close_time, '9999-12-31 23:59:59+00:00')) DESC, start_time DESC, run_id);