[MM-53291] Data retention improvements cherry pick 7.8 (#24492)

* cherry picking data retention fix and fixing conflicts
---------
Co-authored-by: Mattermost Build <build@mattermost.com>

app/user.go

@@ -1661,6 +1661,10 @@ func (a *App) PermanentDeleteUser(c *request.Context, user *model.User) *model.A
 		return model.NewAppError("PermanentDeleteUser", "app.post.permanent_delete_by_user.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
 
+	if err := a.Srv().Store().Reaction().PermanentDeleteByUser(user.Id); err != nil {
+		return model.NewAppError("PermanentDeleteUser", "app.reaction.permanent_delete_by_user.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
 	if err := a.Srv().Store().Bot().PermanentDelete(user.Id); err != nil {
 		var invErr *store.ErrInvalidInput
 		switch {

db/migrations/migrations.list

@@ -206,6 +206,8 @@ db/migrations/mysql/000105_remove_tokens.down.sql
 db/migrations/mysql/000105_remove_tokens.up.sql
 db/migrations/mysql/000107_threadmemberships_cleanup.down.sql
 db/migrations/mysql/000107_threadmemberships_cleanup.up.sql
+db/migrations/mysql/000113_create_retentionidsfordeletion_table.down.sql
+db/migrations/mysql/000113_create_retentionidsfordeletion_table.up.sql
 db/migrations/postgres/000001_create_teams.down.sql
 db/migrations/postgres/000001_create_teams.up.sql
 db/migrations/postgres/000002_create_team_members.down.sql
@@ -412,3 +414,5 @@ db/migrations/postgres/000105_remove_tokens.down.sql
 db/migrations/postgres/000105_remove_tokens.up.sql
 db/migrations/postgres/000107_threadmemberships_cleanup.down.sql
 db/migrations/postgres/000107_threadmemberships_cleanup.up.sql
+db/migrations/postgres/000113_create_retentionidsfordeletion_table.down.sql
+db/migrations/postgres/000113_create_retentionidsfordeletion_table.up.sql

db/migrations/mysql/000113_create_retentionidsfordeletion_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS RetentionIdsForDeletion;

db/migrations/mysql/000113_create_retentionidsfordeletion_table.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS RetentionIdsForDeletion (
+    Id varchar(26) NOT NULL,
+    TableName varchar(64),
+    Ids json,
+    PRIMARY KEY (Id),
+    KEY idx_retentionidsfordeletion_tablename (TableName)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

db/migrations/postgres/000113_create_retentionidsfordeletion_table.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS idx_retentionidsfordeletion_tablename;
+
+DROP TABLE IF EXISTS retentionidsfordeletion;

db/migrations/postgres/000113_create_retentionidsfordeletion_table.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS retentionidsfordeletion (
+    id varchar(26) PRIMARY KEY,
+    tablename varchar(64),
+    ids varchar(26)[]
+);
+
+CREATE INDEX IF NOT EXISTS idx_retentionidsfordeletion_tablename ON retentionidsfordeletion (tablename);

i18n/en.json

@@ -6299,6 +6299,10 @@
     "id": "app.reaction.get_for_post.app_error",
     "translation": "Unable to get reactions for post."
   },
+  {
+    "id": "app.reaction.permanent_delete_by_user.app_error",
+    "translation": "Unable to delete reactions for user."
+  },
   {
     "id": "app.reaction.save.save.app_error",
     "translation": "Unable to save reaction."

model/config.go

@@ -210,6 +210,7 @@ const (
 	DataRetentionSettingsDefaultDeletionJobStartTime           = "02:00"
 	DataRetentionSettingsDefaultBatchSize                      = 3000
 	DataRetentionSettingsDefaultTimeBetweenBatchesMilliseconds = 100
+	DataRetentionSettingsDefaultRetentionIdsBatchSize          = 100
 
 	PluginSettingsDefaultDirectory         = "./plugins"
 	PluginSettingsDefaultClientDirectory   = "./client/plugins"
@@ -2716,6 +2717,7 @@ type DataRetentionSettings struct {
 	DeletionJobStartTime           *string `access:"compliance_data_retention_policy"`
 	BatchSize                      *int    `access:"compliance_data_retention_policy"`
 	TimeBetweenBatchesMilliseconds *int    `access:"compliance_data_retention_policy"`
+	RetentionIdsBatchSize          *int    `access:"compliance_data_retention_policy"`
 }
 
 func (s *DataRetentionSettings) SetDefaults() {
@@ -2754,6 +2756,9 @@ func (s *DataRetentionSettings) SetDefaults() {
 	if s.TimeBetweenBatchesMilliseconds == nil {
 		s.TimeBetweenBatchesMilliseconds = NewInt(DataRetentionSettingsDefaultTimeBetweenBatchesMilliseconds)
 	}
+	if s.RetentionIdsBatchSize == nil {
+		s.RetentionIdsBatchSize = NewInt(DataRetentionSettingsDefaultRetentionIdsBatchSize)
+	}
 }
 
 type JobSettings struct {

model/data_retention_policy.go

@@ -86,3 +86,15 @@ type RetentionPolicyCursor struct {
 	TeamPoliciesDone    bool
 	GlobalPoliciesDone  bool
 }
+
+type RetentionIdsForDeletion struct {
+	Id        string
+	TableName string
+	Ids       []string
+}
+
+func (r *RetentionIdsForDeletion) PreSave() {
+	if r.Id == "" {
+		r.Id = NewId()
+	}
+}

model/feature_flags.go

@@ -78,8 +78,6 @@ type FeatureFlags struct {
 	GlobalDrafts bool
 
 	OnboardingTourTips bool
-
-	DataRetentionConcurrencyEnabled bool
 }
 
 func (f *FeatureFlags) SetDefaults() {
@@ -110,7 +108,6 @@ func (f *FeatureFlags) SetDefaults() {
 	f.GlobalDrafts = true
 	f.WysiwygEditor = false
 	f.OnboardingTourTips = true
-	f.DataRetentionConcurrencyEnabled = true
 }
 
 func (f *FeatureFlags) Plugins() map[string]string {

services/telemetry/telemetry.go

@@ -795,6 +795,7 @@ func (ts *TelemetryService) trackConfig() {
 		"deletion_job_start_time":       *cfg.DataRetentionSettings.DeletionJobStartTime,
 		"batch_size":                    *cfg.DataRetentionSettings.BatchSize,
 		"time_between_batches":          *cfg.DataRetentionSettings.TimeBetweenBatchesMilliseconds,
+		"retention_ids_batch_size":      *cfg.DataRetentionSettings.RetentionIdsBatchSize,
 		"cleanup_jobs_threshold_days":   *cfg.JobSettings.CleanupJobsThresholdDays,
 		"cleanup_config_threshold_days": *cfg.JobSettings.CleanupConfigThresholdDays,
 	})

store/sqlstore/channel_member_history_store.go

@@ -175,6 +175,7 @@ func (s SqlChannelMemberHistoryStore) PermanentDeleteBatchForRetentionPolicies(n
 		NowMillis:           now,
 		GlobalPolicyEndTime: globalPolicyEndTime,
 		Limit:               limit,
+		StoreDeletedIds:     false,
 	}, s.SqlStore, cursor)
 }