0005658: Snaphot util too slow for large multi-tenant deployment

JumpMind · Jan 19, 2023 · a0cc5e3 · a0cc5e3
1 parent 66fca87
commit a0cc5e3
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 0 deletions.
diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java
@@ -45,10 +45,12 @@
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.TimeZone;
 
 import javax.management.MBeanServer;
@@ -130,6 +132,7 @@ public static File createSnapshot(ISymmetricEngine engine, IProgressListener lis
         dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
         String dirName = engine.getEngineName().replaceAll(" ", "-") + "-" + dateFormat.format(new Date());
         IParameterService parameterService = engine.getParameterService();
+        long timeoutMillis = parameterService.getLong(ParameterConstants.SNAPSHOT_OPERATION_TIMEOUT_MS, 30000);
         File tmpDir = new File(parameterService.getTempDirectory(), dirName);
         tmpDir.mkdirs();
         log.info("Creating snapshot file in " + tmpDir.getAbsolutePath());
@@ -164,13 +167,27 @@ public static File createSnapshot(ISymmetricEngine engine, IProgressListener lis
             ITriggerRouterService triggerRouterService = engine.getTriggerRouterService();
             List<TriggerHistory> triggerHistories = triggerRouterService.getActiveTriggerHistories();
             String tablePrefix = engine.getTablePrefix().toUpperCase();
+            Set<String> triggerIds = new HashSet<String>();
+            boolean isClonedTables = parameterService.is("sync.triggers.expand.table.clone", true);
+            long ts = System.currentTimeMillis();
             for (TriggerHistory triggerHistory : triggerHistories) {
                 if (!triggerHistory.getSourceTableName().toUpperCase().startsWith(tablePrefix)) {
+                    if (isClonedTables && !triggerIds.add(triggerHistory.getTriggerId())) {
+                        Trigger trigger = triggerRouterService.getTriggerById(triggerHistory.getTriggerId(), false);
+                        if (trigger != null && trigger.getSourceTableName().contains("$(targetExternalId)")) {
+                            // for multi-tenant database where the same table is repeated for each node, just need one definition
+                            continue;
+                        }
+                    }
                     Table table = targetPlatform.getTableFromCache(triggerHistory.getSourceCatalogName(),
                             triggerHistory.getSourceSchemaName(), triggerHistory.getSourceTableName(), false);
                     if (table != null) {
                         addTableToMap(catalogSchemas, new CatalogSchema(table.getCatalog(), table.getSchema()), table);
                     }
+                    if (System.currentTimeMillis() - ts > timeoutMillis) {
+                        log.info("Reached time limit for table definitions");
+                        break;
+                    }
                 }
             }
             addTablesThatLoadIncoming(engine, catalogSchemas);

diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
@@ -537,6 +537,7 @@ private ParameterConstants() {
     public final static String SNAPSHOT_MAX_FILES = "snapshot.max.files";
     public final static String SNAPSHOT_MAX_BATCHES = "snapshot.max.batches";
     public final static String SNAPSHOT_MAX_NODE_CHANNELS = "snapshot.max.node.channels";
+    public final static String SNAPSHOT_OPERATION_TIMEOUT_MS = "snapshot.operation.timeout.ms";
 
     public final static String REDSHIFT_APPEND_TO_COPY_COMMAND = "redshift.append.to.copy.command";
     public final static String REDSHIFT_BULK_LOAD_MAX_ROWS_BEFORE_FLUSH = "redshift.bulk.load.max.rows.before.flush";

diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties
@@ -3040,6 +3040,14 @@ snapshot.max.batches=10000
 # Type: integer
 snapshot.max.node.channels=5000
 
+# Max time for a snapshot operation to complete, such as gathering table definitions,
+# before it will be interrupted so the snapshot completes in a reasonable amount of time.
+#
+# DatabaseOverridable: true
+# Tags: other
+# Type: integer
+snapshot.operation.timeout.ms=30000
+
 # Log Miner job to find changes from a database archive log
 #
 # DatabaseOverridable: false