From e70e857e1d63f8dc46a653bd8ff685fc225d10ce Mon Sep 17 00:00:00 2001
From: Eric Long <elong@jumpmind.com>
Date: Thu, 19 Jan 2023 16:06:14 -0500
Subject: [PATCH] 0005659: Snapshot util too slow for large multi-tenant
 deployment

---
 .../jumpmind/symmetric/util/SnapshotUtil.java   | 17 +++++++++++++++++
 .../symmetric/common/ParameterConstants.java    |  1 +
 .../main/resources/symmetric-default.properties |  8 ++++++++
 3 files changed, 26 insertions(+)

diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java
index 3e8a33740f..2e60b60025 100644
--- a/symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java
+++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java
@@ -45,10 +45,12 @@
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.TimeZone;
 
 import javax.management.MBeanServer;
@@ -130,6 +132,7 @@ public static File createSnapshot(ISymmetricEngine engine, IProgressListener lis
         dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
         String dirName = engine.getEngineName().replaceAll(" ", "-") + "-" + dateFormat.format(new Date());
         IParameterService parameterService = engine.getParameterService();
+        long timeoutMillis = parameterService.getLong(ParameterConstants.SNAPSHOT_OPERATION_TIMEOUT_MS, 30000);
         File tmpDir = new File(parameterService.getTempDirectory(), dirName);
         tmpDir.mkdirs();
         log.info("Creating snapshot file in " + tmpDir.getAbsolutePath());
@@ -164,13 +167,27 @@ public static File createSnapshot(ISymmetricEngine engine, IProgressListener lis
             ITriggerRouterService triggerRouterService = engine.getTriggerRouterService();
             List<TriggerHistory> triggerHistories = triggerRouterService.getActiveTriggerHistories();
             String tablePrefix = engine.getTablePrefix().toUpperCase();
+            Set<String> triggerIds = new HashSet<String>();
+            boolean isClonedTables = parameterService.is("sync.triggers.expand.table.clone", true);
+            long ts = System.currentTimeMillis();
             for (TriggerHistory triggerHistory : triggerHistories) {
                 if (!triggerHistory.getSourceTableName().toUpperCase().startsWith(tablePrefix)) {
+                    if (isClonedTables && !triggerIds.add(triggerHistory.getTriggerId())) {
+                        Trigger trigger = triggerRouterService.getTriggerById(triggerHistory.getTriggerId(), false);
+                        if (trigger != null && trigger.getSourceTableName().contains("$(targetExternalId)")) {
+                            // for multi-tenant database where the same table is repeated for each node, just need one definition
+                            continue;
+                        }
+                    }
                     Table table = targetPlatform.getTableFromCache(triggerHistory.getSourceCatalogName(),
                             triggerHistory.getSourceSchemaName(), triggerHistory.getSourceTableName(), false);
                     if (table != null) {
                         addTableToMap(catalogSchemas, new CatalogSchema(table.getCatalog(), table.getSchema()), table);
                     }
+                    if (System.currentTimeMillis() - ts > timeoutMillis) {
+                        log.info("Reached time limit for table definitions");
+                        break;
+                    }
                 }
             }
             addTablesThatLoadIncoming(engine, catalogSchemas);
diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
index c886ebc4b0..043da8ab95 100644
--- a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
+++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
@@ -435,6 +435,7 @@ private ParameterConstants() {
     public final static String SNAPSHOT_MAX_FILES = "snapshot.max.files";
     public final static String SNAPSHOT_MAX_BATCHES = "snapshot.max.batches";
     public final static String SNAPSHOT_MAX_NODE_CHANNELS = "snapshot.max.node.channels";
+    public final static String SNAPSHOT_OPERATION_TIMEOUT_MS = "snapshot.operation.timeout.ms";
     public final static String REDSHIFT_APPEND_TO_COPY_COMMAND = "redshift.append.to.copy.command";
     public final static String REDSHIFT_BULK_LOAD_MAX_ROWS_BEFORE_FLUSH = "redshift.bulk.load.max.rows.before.flush";
     public final static String REDSHIFT_BULK_LOAD_MAX_BYTES_BEFORE_FLUSH = "redshift.bulk.load.max.bytes.before.flush";
diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties
index 5d1e449a6e..9c6acb5f75 100644
--- a/symmetric-core/src/main/resources/symmetric-default.properties
+++ b/symmetric-core/src/main/resources/symmetric-default.properties
@@ -3067,6 +3067,14 @@ snapshot.max.batches=10000
 # Type: integer
 snapshot.max.node.channels=5000
 
+# Max time for a snapshot operation to complete, such as gathering table definitions,
+# before it will be interrupted so the snapshot completes in a reasonable amount of time.
+#
+# DatabaseOverridable: true
+# Tags: other
+# Type: integer
+snapshot.operation.timeout.ms=30000
+
 # Log Miner job to find changes from a database archive log
 #
 # DatabaseOverridable: false