From 706f79fdb2b580565409fb04e5cc10d2ca59373d Mon Sep 17 00:00:00 2001 From: Eric Long Date: Thu, 14 May 2020 08:41:09 -0400 Subject: [PATCH 01/10] 0004374: back out template change from 3.11... will make on 3.12 --- .../org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java index 4a972b3488..4e00f15bc6 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java @@ -43,7 +43,7 @@ public OracleTriggerTemplate(ISymmetricDialect symmetricDialect) { dateTimeWithLocalTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; timeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; dateColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; - clobColumnTemplate = "case when $(tableAlias).\"$(columnName)\" is null then null else '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"' end" ; + clobColumnTemplate = "decode(dbms_lob.getlength(to_clob($(tableAlias).\"$(columnName)\")), null, to_clob(''), '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"')" ; blobColumnTemplate = "decode(dbms_lob.getlength($(tableAlias).\"$(columnName)\"), null, to_clob(''), '\"'||$(prefixName)_blob2clob($(tableAlias).\"$(columnName)\")||'\"')" ; longColumnTemplate = "$(oracleToClob)'\"\\b\"'"; booleanColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', '\"'||cast($(tableAlias).\"$(columnName)\" as number("+symmetricDialect.getTemplateNumberPrecisionSpec()+"))||'\"')" ; From f8fac7086bc914f7ce2b36d90f08a1017e090a19 Mon Sep 17 00:00:00 2001 From: Eric Long Date: Wed, 20 May 2020 09:17:38 -0400 Subject: [PATCH 02/10] 0004401: Oracle ntype characters lost from conversion in capture --- .../db/oracle/OracleTriggerTemplate.java | 24 +++++++++---------- .../symmetric/db/AbstractTriggerTemplate.java | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java index 4a972b3488..2ea3b732b1 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java @@ -35,19 +35,19 @@ public OracleTriggerTemplate(ISymmetricDialect symmetricDialect) { // @formatter:off emptyColumnTemplate = "''" ; - stringColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, $(oracleToClob)'', '\"'||replace(replace($(oracleToClob)$(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"')" ; - geometryColumnTemplate = "case when $(tableAlias).\"$(columnName)\" is null then to_clob('') else '\"'||replace(replace(SDO_UTIL.TO_WKTGEOMETRY($(tableAlias).\"$(columnName)\"),'\\','\\\\'),'\"','\\\"')||'\"' end"; - numberColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', '\"'||" + getNumberConversionString() + "||'\"')" ; - datetimeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; - dateTimeWithTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM')),'\"'))" ; - dateTimeWithLocalTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; - timeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; - dateColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; - clobColumnTemplate = "case when $(tableAlias).\"$(columnName)\" is null then null else '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"' end" ; - blobColumnTemplate = "decode(dbms_lob.getlength($(tableAlias).\"$(columnName)\"), null, to_clob(''), '\"'||$(prefixName)_blob2clob($(tableAlias).\"$(columnName)\")||'\"')" ; + stringColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", '\"'||replace(replace($(oracleToClob)$(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"', '')"; + geometryColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", '\"'||replace(replace(SDO_UTIL.TO_WKTGEOMETRY($(tableAlias).\"$(columnName)\"),'\\','\\\\'),'\"','\\\"')||'\"', '')"; + numberColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", $(oracleToClob)'\"'||" + getNumberConversionString() + "||'\"', '')"; + datetimeColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", $(oracleToClob)concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'), '')"; + dateTimeWithTimeZoneColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", $(oracleToClob)concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM')),'\"'), '')"; + dateTimeWithLocalTimeZoneColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", $(oracleToClob)concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'), '')"; + timeColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", $(oracleToClob)concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'), '')"; + dateColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", $(oracleToClob)concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'), '')"; + clobColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"', '')"; + blobColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", '\"'||$(prefixName)_blob2clob($(tableAlias).\"$(columnName)\")||'\"', '')"; longColumnTemplate = "$(oracleToClob)'\"\\b\"'"; - booleanColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', '\"'||cast($(tableAlias).\"$(columnName)\" as number("+symmetricDialect.getTemplateNumberPrecisionSpec()+"))||'\"')" ; - xmlColumnTemplate = "decode(dbms_lob.getlength(extract($(tableAlias).\"$(columnName)\", '/').getclobval()), null, to_clob(''), '\"'||replace(replace(extract($(tableAlias).\"$(columnName)\", '/').getclobval(),'\\','\\\\'),'\"','\\\"')||'\"')" ; + booleanColumnTemplate = "nvl2($(tableAlias).\"$(columnName)\", '\"'||cast($(tableAlias).\"$(columnName)\" as number("+symmetricDialect.getTemplateNumberPrecisionSpec()+"))||'\"', '')"; + xmlColumnTemplate = "nvl2(extract($(tableAlias).\"$(columnName)\", '/').getclobval(), '\"'||replace(replace(extract($(tableAlias).\"$(columnName)\", '/').getclobval(),'\\','\\\\'),'\"','\\\"')||'\"', '')"; binaryColumnTemplate = blobColumnTemplate; triggerConcatCharacter = "||" ; newTriggerValue = ":new" ; diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java index 7a15a1d1a5..271711fb5e 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java @@ -544,7 +544,8 @@ protected String replaceTemplateVariables(DataEventType dml, Trigger trigger, ddl = FormatUtils.replace("sourceNodeExpression", symmetricDialect.getSourceNodeExpression(), ddl); - ddl = FormatUtils.replace("oracleLobType", trigger.isUseCaptureLobs() ? getClobType(table) : "long", + ddl = FormatUtils.replace("oracleLobType", trigger.isUseCaptureLobs() ? getClobType(table) : + symmetricDialect.getParameterService().is(ParameterConstants.DBDIALECT_ORACLE_USE_NTYPES_FOR_SYNC) ? "NVARCHAR2(4000)" : "VARCHAR2(4000)", ddl); ddl = FormatUtils.replace("oracleLobTypeClobAlways", getClobType(table), ddl); From 44539ea87d28096dbf332c67e034149d8bd82050 Mon Sep 17 00:00:00 2001 From: Eric Long Date: Wed, 20 May 2020 13:17:07 -0400 Subject: [PATCH 03/10] 0004376: Monitor 'offlineNodes' does not capture offline nodes in detail --- .../org/jumpmind/symmetric/monitor/MonitorTypeOfflineNodes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeOfflineNodes.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeOfflineNodes.java index 56a1d44786..7b9081427a 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeOfflineNodes.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeOfflineNodes.java @@ -72,7 +72,7 @@ public void setSymmetricEngine(ISymmetricEngine engine) { protected String serializeDetails(List offlineNodes) { String result = null; try { - new Gson().toJson(offlineNodes); + result = new Gson().toJson(offlineNodes); } catch(Exception e) { log.warn("Unable to convert list of offline nodes to JSON", e); } From 6ea3a61f40d3e74cee0d8417cb811ec2db7df909 Mon Sep 17 00:00:00 2001 From: Eric Long Date: Wed, 20 May 2020 14:41:26 -0400 Subject: [PATCH 04/10] 0004403: Authentication interceptor out of memory --- .../symmetric/common/ParameterConstants.java | 1 + .../resources/symmetric-default.properties | 9 ++++ .../web/AuthenticationInterceptor.java | 44 +++++++++++++++++-- .../symmetric/web/ServerSymmetricEngine.java | 3 +- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java index ea50267a81..8940150da5 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java @@ -286,6 +286,7 @@ private ParameterConstants() { public final static String TRANSPORT_HTTP_COMPRESSION_STRATEGY = "compression.strategy"; public final static String TRANSPORT_HTTP_USE_SESSION_AUTH = "http.use.session.auth"; public final static String TRANSPORT_HTTP_SESSION_EXPIRE_SECONDS = "http.session.expire.seconds"; + public final static String TRANSPORT_HTTP_SESSION_MAX_COUNT = "http.session.max.count"; public final static String TRANSPORT_HTTP_USE_HEADER_SECURITY_TOKEN = "http.use.header.security.token"; public final static String TRANSPORT_TYPE = "transport.type"; public final static String TRANSPORT_MAX_BYTES_TO_SYNC = "transport.max.bytes.to.sync"; diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties index 960b611d82..042307ec3f 100644 --- a/symmetric-core/src/main/resources/symmetric-default.properties +++ b/symmetric-core/src/main/resources/symmetric-default.properties @@ -435,6 +435,15 @@ http.use.session.auth=true # Type: integer http.session.expire.seconds=14400 +# Maximum number of authenticated sessions to keep in memory before removing the oldest. +# Normally, this won't be reached unless something is mis-configured, like a cluster that is +# not using a sticky session load balancer. +# +# DatabaseOverridable: false +# Tags: transport +# Type: integer +http.session.max.count=15000 + # When authenticating to a server node, send the security token in the request header instead # of using a URL parameter. Using the request header avoids accidentally logging the # security token. The transport uses the remote node's version to determine if it should use header or parameter. diff --git a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/AuthenticationInterceptor.java b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/AuthenticationInterceptor.java index 354669426f..00253433b0 100644 --- a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/AuthenticationInterceptor.java +++ b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/AuthenticationInterceptor.java @@ -21,8 +21,9 @@ package org.jumpmind.symmetric.web; import java.io.IOException; -import java.util.HashMap; +import java.util.Iterator; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -30,6 +31,7 @@ import org.apache.commons.lang.StringUtils; import org.jumpmind.security.ISecurityService; +import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.service.INodeService; import org.jumpmind.symmetric.service.INodeService.AuthenticationStatus; import org.slf4j.Logger; @@ -46,18 +48,23 @@ public class AuthenticationInterceptor implements IInterceptor { private ISecurityService securityService; - private Map sessions = new HashMap(); + private Map sessions = new ConcurrentHashMap(); private boolean useSessionAuth; private int sessionExpireMillis; + private int maxSessions; + + private long maxSessionsLastTime; + public AuthenticationInterceptor(INodeService nodeService, ISecurityService securityService, - boolean useSessionAuth, int sessionExpireSeconds) { + boolean useSessionAuth, int sessionExpireSeconds, int maxSessions) { this.nodeService = nodeService; this.securityService = securityService; this.useSessionAuth = useSessionAuth; this.sessionExpireMillis = sessionExpireSeconds * 1000; + this.maxSessions = maxSessions; } public boolean before(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException { @@ -124,14 +131,43 @@ public boolean before(HttpServletRequest req, HttpServletResponse resp) throws I protected AuthenticationSession getSession(HttpServletRequest req, boolean create) { String sessionId = req.getHeader(WebConstants.HEADER_SESSION_ID); - AuthenticationSession session = sessions.get(sessionId); + AuthenticationSession session = null; + if (sessionId != null) { + session = sessions.get(sessionId); + } if (session == null && create) { + if (sessions.size() >= maxSessions) { + removeOldSessions(); + } String id = securityService.nextSecureHexString(30); session = new AuthenticationSession(id); sessions.put(id, session); } return session; } + + protected void removeOldSessions() { + long now = System.currentTimeMillis(); + int removedSessions = 0; + AuthenticationSession oldestSession = null; + Iterator iter = sessions.values().iterator(); + while (iter.hasNext()) { + AuthenticationSession session = iter.next(); + if (now - session.getCreationTime() > sessionExpireMillis) { + iter.remove(); + removedSessions++; + } else if (oldestSession == null || session.getCreationTime() < oldestSession.getCreationTime()) { + oldestSession = session; + } + } + if (removedSessions == 0 && oldestSession != null) { + sessions.remove(oldestSession.getId()); + } + if (maxSessionsLastTime == 0 || now - maxSessionsLastTime > 60000) { + maxSessionsLastTime = now; + log.warn("Max node authentication sessions reached, removing old sessions. See parameter " + ParameterConstants.TRANSPORT_HTTP_SESSION_MAX_COUNT); + } + } public void after(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException { } diff --git a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/ServerSymmetricEngine.java b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/ServerSymmetricEngine.java index d1dfaeeea6..8c195ef748 100644 --- a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/ServerSymmetricEngine.java +++ b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/ServerSymmetricEngine.java @@ -81,7 +81,8 @@ protected void init() { boolean useSessionAuth = parameterService.is(ParameterConstants.TRANSPORT_HTTP_USE_SESSION_AUTH); int sessionExpireSeconds = parameterService.getInt(ParameterConstants.TRANSPORT_HTTP_SESSION_EXPIRE_SECONDS); - AuthenticationInterceptor authInterceptor = new AuthenticationInterceptor(nodeService, securityService, useSessionAuth, sessionExpireSeconds); + int maxSessions = parameterService.getInt(ParameterConstants.TRANSPORT_HTTP_SESSION_MAX_COUNT, 1000); + AuthenticationInterceptor authInterceptor = new AuthenticationInterceptor(nodeService, securityService, useSessionAuth, sessionExpireSeconds, maxSessions); NodeConcurrencyInterceptor concurrencyInterceptor = new NodeConcurrencyInterceptor( concurrentConnectionManager, configurationService, statisticManager); IInterceptor[] customInterceptors = buildCustomInterceptors(); From 8088937b1186794c992691935fbe2ce5ff02e27a Mon Sep 17 00:00:00 2001 From: Eric Long Date: Thu, 21 May 2020 10:55:48 -0400 Subject: [PATCH 05/10] 0004226: trigger template variables and example --- .../asciidoc/configuration/table-triggers.ad | 17 ++++++++------- .../table-triggers/external-select.ad | 17 ++------------- .../table-triggers/trigger-variables.ad | 21 +++++++++++++++++++ 3 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 symmetric-assemble/src/asciidoc/configuration/table-triggers/trigger-variables.ad diff --git a/symmetric-assemble/src/asciidoc/configuration/table-triggers.ad b/symmetric-assemble/src/asciidoc/configuration/table-triggers.ad index c7329622d9..692b09506c 100644 --- a/symmetric-assemble/src/asciidoc/configuration/table-triggers.ad +++ b/symmetric-assemble/src/asciidoc/configuration/table-triggers.ad @@ -55,19 +55,19 @@ ifdef::pro[] .Advanced Options endif::pro[] -Sync On Insert:: Determines if changes will be captured for inserts. -Sync On Update:: Determines if changes will be captured for updates. -Sync On Delete:: Determines if changes will be captured for deletes. +Sync On Insert:: Flag for installing an insert trigger. +Sync On Update:: Flag for installing an update trigger. +Sync On Delete:: Flag for installing a delete trigger. Reload Channel Id:: The channel_id of the channel that will be used for initial loads. -Sync On Insert Condition:: Specify a condition for the insert trigger firing using an expression specific to the database. On most platforms, it is added to an "IF" statement in the trigger text. On SQL-Server it is added to the "WHERE" clause of a query for inserted/deleted logical tables. See Sync Condition Example. -Sync On Update Condition:: Specify a condition for the update trigger firing using an expression specific to the database. On most platforms, it is added to an "IF" statement in the trigger text. On SQL-Server it is added to the "WHERE" clause of a query for inserted/deleted logical tables. See Sync Condition Example. -Sync On Delete Condition:: Specify a condition for the delete trigger firing using an expression specific to the database. On most platforms, it is added to an "IF" statement in the trigger text. On SQL-Server it is added to the "WHERE" clause of a query for inserted/deleted logical tables. See Sync Condition Example. -Sync Condition Example:: Sync Conditions can access both old values and new values of a field/column using "old_" and "new_" respectively. For example, if your column is id and your condition checks the value coming in to be 'test', your condition will be: +Sync Conditions:: A procedure language expression included in the trigger text to determine whether a change is captured or not. Most platforms include the condition inside an "IF" statement, while SQL-Server includes the condition in a "WHERE" clause. Old and new values of a column can be referenced using "$(oldTriggerValue)" and "$(newTriggerValue)" aliases respectively. See <>. For example, if a character column is named "STATUS" and the row should be captured when the value is "2", then the condition would be: + ---- - new_id = 'test' + $(newTriggerValue).status = '2' ---- +Sync On Insert Condition:: Conditional expression for the insert trigger to determine if a change is captured or not. See Sync Conditions. +Sync On Update Condition:: Conditional expression for the update trigger to determine if a change is captured or not. See Sync Conditions. +Sync On Delete Condition:: Conditional expression for the delete trigger to determine if a change is captured or not. See Sync Conditions. Custom Insert Trigger Text:: Specify insert trigger text (SQL) to execute after the SymmetricDS trigger fires. This field is not applicable for H2, HSQLDB 1.x or Apache Derby. Custom Update Trigger Text:: Specify update trigger text (SQL) to execute after the SymmetricDS trigger fires. This field is not applicable for H2, HSQLDB 1.x or Apache Derby. Custom Delete Trigger Text:: Specify delete trigger text (SQL) to execute after the SymmetricDS trigger fires. This field is not applicable for H2, HSQLDB 1.x or Apache Derby. @@ -129,4 +129,5 @@ This property is currently only supported on MySQL, DB2, SQL Server, and Oracle. include::table-triggers/wildcards.ad[] include::table-triggers/external-select.ad[] +include::table-triggers/trigger-variables.ad[] include::table-triggers/load-only.ad[] \ No newline at end of file diff --git a/symmetric-assemble/src/asciidoc/configuration/table-triggers/external-select.ad b/symmetric-assemble/src/asciidoc/configuration/table-triggers/external-select.ad index 25049ca99b..f54e281435 100644 --- a/symmetric-assemble/src/asciidoc/configuration/table-triggers/external-select.ad +++ b/symmetric-assemble/src/asciidoc/configuration/table-triggers/external-select.ad @@ -6,7 +6,8 @@ This data is typically needed for the purposes of determining where to 'route' t definition contains an optional "external select" field which can be used to specify the data to be captured. Once captured, this data is available during routing in DATA 's external_data field. -For these cases, place a SQL select statement which returns the data item you need for routing in external_select. +For these cases, place a SQL select statement which returns the data item you need for routing in external_select. +See <> for a list of variables available for use. IMPORTANT: The external select SQL must return a single row, single column @@ -32,20 +33,6 @@ values ('orderlineitem', 'orderlineitem','orderlineitem', ==== endif::pro[] -.The following variables can be used with the external select -[cols=".^2,8"] -|=== - -|$(curTriggerValue)|Variable to be replaced with the NEW or OLD column alias provided by the trigger context, which is platform specific. -For insert and update triggers, the NEW alias is used; for delete triggers, the OLD alias is used. For example, "$(curTriggerValue).COLUMN" -becomes ":new.COLUMN" for an insert trigger on Oracle. - -|$(curColumnPrefix)|Variable to be replaced with the NEW_ or OLD_ column prefix for platforms that don't support column aliases. This is -currently only used by the H2 database. All other platforms will replace the variable with an empty string. For example "$(curColumnPrefix)COLUMN" -becomes "NEW_COLUMN" on H2 and "COLUMN" on Oracle. - -|=== - WARNING: External select SQL statements should be used carefully as they will cause the trigger to run the additional SQL each time the trigger fires. TIP: Using an external select on the trigger is similar to using the 'subselect' router. The advantage of this approach over the 'subselect' approach diff --git a/symmetric-assemble/src/asciidoc/configuration/table-triggers/trigger-variables.ad b/symmetric-assemble/src/asciidoc/configuration/table-triggers/trigger-variables.ad new file mode 100644 index 0000000000..2ecd95c98d --- /dev/null +++ b/symmetric-assemble/src/asciidoc/configuration/table-triggers/trigger-variables.ad @@ -0,0 +1,21 @@ + +==== Trigger Variables + +The Sync Condition, External Select, and Custom Trigger Text configurations allow the user to provide +procedure language text that is included inside the trigger. +Variables can be used for configuration that works across different database platforms. +When triggers are created, the variables are replaced with the syntax needed for that specific database. + +.Trigger Template Variables +[cols=".^2,8"] +|=== + +|$(newTriggerValue)|New row alias for inserts and updates. For example, "$(newTriggerValue).MYCOLUMN" becomes ":new.MYCOLUMN" for an insert/update trigger on Oracle. + +|$(oldTriggerValue)|Old row alias for updates and deletes. For example, "$(oldTriggerValue).MYCOLUMN" becomes ":old.MYCOLUMN" for an update/delete trigger on Oracle. + +|$(curTriggerValue)|Current row alias for insert, updates, and deletes. This variable acts like $(newTriggerValue) for inserts and updates, and it acts like $(oldTriggerValue) for deletes. + +|$(curColumnPrefix)|Column prefix only used by H2 database. It is replaced with the NEW_ or OLD_ column prefix needed by H2. All other platforms will replace the variable with an empty string + +|=== From 52d93e6d6ec0532fd10101d36604db5b8207b153 Mon Sep 17 00:00:00 2001 From: Eric Long Date: Thu, 21 May 2020 11:05:38 -0400 Subject: [PATCH 06/10] 0004229: [Docs] Invalid column name in "Example 2. Sample Group Links" --- symmetric-assemble/src/asciidoc/configuration/group-links.ad | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-assemble/src/asciidoc/configuration/group-links.ad b/symmetric-assemble/src/asciidoc/configuration/group-links.ad index a0640ebfbd..755e383ead 100644 --- a/symmetric-assemble/src/asciidoc/configuration/group-links.ad +++ b/symmetric-assemble/src/asciidoc/configuration/group-links.ad @@ -43,11 +43,11 @@ ifndef::pro[] [source,sql] ---- insert into SYM_NODE_GROUP_LINK -(source_node_group, target_node_group, data_event_action) +(source_node_group_id, target_node_group_id, data_event_action) values ('store', 'corp', 'P'); insert into SYM_NODE_GROUP_LINK -(source_node_group, target_node_group, data_event_action) +(source_node_group_id, target_node_group_id, data_event_action) values ('corp', 'store', 'W'); ---- endif::pro[] From 5c80717cc67117a05c780937cc1aa7cfd936a8f8 Mon Sep 17 00:00:00 2001 From: Eric Long Date: Thu, 21 May 2020 11:14:43 -0400 Subject: [PATCH 07/10] 0004225: fix npe converting to date --- .../jumpmind/db/platform/sqlite/SqliteJdbcSqlTemplate.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/sqlite/SqliteJdbcSqlTemplate.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/sqlite/SqliteJdbcSqlTemplate.java index d58520ab5d..20e3747a83 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/sqlite/SqliteJdbcSqlTemplate.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/sqlite/SqliteJdbcSqlTemplate.java @@ -75,9 +75,11 @@ public T getObjectFromResultSet(ResultSet rs, Class clazz) throws SQLExce String s = rs.getString(1); Date d = null; - d = FormatUtils.parseDate(s,FormatUtils.TIMESTAMP_PATTERNS); + if (s != null) { + d = FormatUtils.parseDate(s, FormatUtils.TIMESTAMP_PATTERNS); + } - if (Timestamp.class.isAssignableFrom(clazz)) { + if (d != null && Timestamp.class.isAssignableFrom(clazz)) { return (T) new Timestamp(d.getTime()); } else { return (T) d; From e5534c7950c27c6274623649f854de84a25f1f3a Mon Sep 17 00:00:00 2001 From: Eric Long Date: Thu, 21 May 2020 11:23:44 -0400 Subject: [PATCH 08/10] 0004398: Initial load leaves batch in RQ status with message "batch not ready for delivery" --- .../jumpmind/symmetric/service/impl/DataExtractorService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java index 26eae2a775..b40e52dd28 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java @@ -1025,13 +1025,13 @@ protected OutgoingBatch extractOutgoingBatch(ProcessInfo extractInfo, Node targe if (currentBatch.getStatus() == Status.IG) { cleanupIgnoredBatch(sourceNode, targetNode, currentBatch, writer); - } else if (!isPreviouslyExtracted(currentBatch, false)) { + } else if (currentBatch.getStatus() == Status.RQ || !isPreviouslyExtracted(currentBatch, false)) { BatchLock lock = null; try { log.debug("{} attempting to acquire lock for batch {}", targetNode.getNodeId(), currentBatch.getBatchId()); lock = acquireLock(currentBatch, useStagingDataWriter); log.debug("{} acquired lock for batch {}", targetNode.getNodeId(), currentBatch.getBatchId()); - if (!isPreviouslyExtracted(currentBatch, true)) { + if (currentBatch.getStatus() == Status.RQ || !isPreviouslyExtracted(currentBatch, true)) { log.debug("{} extracting batch {}", targetNode.getNodeId(), currentBatch.getBatchId()); currentBatch.setExtractCount(currentBatch.getExtractCount() + 1); From 54d2aa4ac344397f72794ac8d455c15aff258599 Mon Sep 17 00:00:00 2001 From: vanmetjk Date: Thu, 21 May 2020 12:07:07 -0400 Subject: [PATCH 09/10] 0004397: Bit Support for Postgresql --- .../db/platform/postgresql/PostgreSqlDdlReader.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDdlReader.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDdlReader.java index 74759c272d..930edb5946 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDdlReader.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDdlReader.java @@ -45,7 +45,6 @@ import org.jumpmind.db.sql.ISqlRowMapper; import org.jumpmind.db.sql.JdbcSqlTemplate; import org.jumpmind.db.sql.Row; -import org.jumpmind.db.sql.SqlTemplateSettings; /* * Reads a database model from a PostgreSql database. @@ -123,7 +122,10 @@ protected Integer mapUnknownJdbcTypeForColumn(Map values) { return Types.BLOB; } else if (type != null && (type == Types.STRUCT || type == Types.OTHER)) { return Types.LONGVARCHAR; - } else { + } else if (typeName != null && typeName.equalsIgnoreCase("BIT")) { + return Types.VARCHAR; + } + else { return super.mapUnknownJdbcTypeForColumn(values); } } From 21ce11289a64833ee2c8335ae448edfdd1baabbe Mon Sep 17 00:00:00 2001 From: Philip Marzullo Date: Fri, 22 May 2020 10:46:04 -0400 Subject: [PATCH 10/10] 0004405: Initial Load: Retrieve active trigger histories after retrieving Sync Trigger lock --- .../symmetric/service/IDataService.java | 8 ++- .../symmetric/service/impl/DataService.java | 12 ++++- .../service/impl/InitialLoadService.java | 53 ++++++++----------- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java index 031a7b767d..e7f118fc34 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java @@ -32,6 +32,7 @@ import org.jumpmind.symmetric.ext.IHeartbeatListener; import org.jumpmind.symmetric.io.data.Batch; import org.jumpmind.symmetric.io.data.CsvData; +import org.jumpmind.symmetric.load.IReloadGenerator; import org.jumpmind.symmetric.model.AbstractBatch.Status; import org.jumpmind.symmetric.model.Data; import org.jumpmind.symmetric.model.DataEvent; @@ -103,8 +104,11 @@ public String reloadTableImmediate(String nodeId, String catalogName, String sch public String sendSQL(String nodeId, String sql); - public Map insertReloadEvents(Node targetNode, boolean reverse, List reloadRequests, ProcessInfo processInfo, - List activeHistories, List triggerRouters, Map extractRequests); + public Map insertReloadEvents( + Node targetNode, boolean reverse, List reloadRequests, + ProcessInfo processInfo, List triggerRouters, + Map extractRequests, + IReloadGenerator reloadGenerator); public boolean insertReloadEvent(TableReloadRequest request, boolean deleteAtClient); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java index a33a7cef51..5efc89ceac 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java @@ -71,6 +71,7 @@ import org.jumpmind.symmetric.io.data.transform.TransformPoint; import org.jumpmind.symmetric.job.OracleNoOrderHeartbeat; import org.jumpmind.symmetric.job.PushHeartbeatListener; +import org.jumpmind.symmetric.load.IReloadGenerator; import org.jumpmind.symmetric.load.IReloadListener; import org.jumpmind.symmetric.load.IReloadVariableFilter; import org.jumpmind.symmetric.model.AbstractBatch.Status; @@ -813,13 +814,22 @@ private String getReloadChannelIdForTrigger(Trigger trigger, Map insertReloadEvents(Node targetNode, boolean reverse, List reloadRequests, ProcessInfo processInfo, - List activeHistories, List triggerRouters, Map extractRequests) { + List triggerRouters, Map extractRequests, + IReloadGenerator reloadGenerator) + { if (engine.getClusterService().lock(ClusterConstants.SYNC_TRIGGERS)) { try { INodeService nodeService = engine.getNodeService(); ITriggerRouterService triggerRouterService = engine.getTriggerRouterService(); synchronized (triggerRouterService) { + + List activeHistories = null; + if (reloadGenerator == null) { + activeHistories = triggerRouterService.getActiveTriggerHistories(); + } else { + activeHistories = reloadGenerator.getActiveTriggerHistories(targetNode); + } boolean isFullLoad = reloadRequests == null || (reloadRequests.size() == 1 && reloadRequests.get(0).isFullLoadRequest()); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/InitialLoadService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/InitialLoadService.java index ba9971e7f7..e7e6c0a9cd 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/InitialLoadService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/InitialLoadService.java @@ -35,7 +35,6 @@ import org.jumpmind.symmetric.model.ExtractRequest; import org.jumpmind.symmetric.model.Node; import org.jumpmind.symmetric.model.NodeGroupLink; -import org.jumpmind.symmetric.model.NodeGroupLinkAction; import org.jumpmind.symmetric.model.NodeSecurity; import org.jumpmind.symmetric.model.ProcessInfo; import org.jumpmind.symmetric.model.ProcessInfoKey; @@ -236,12 +235,8 @@ protected void processTableRequestLoads(Node source, ProcessInfo processInfo) { Map> requestsSplitByLoad = new HashMap>(); Map> triggerRoutersByNodeGroup = new HashMap>(); Map extractRequests = null; - List activeHistories = null; IReloadGenerator reloadGenerator = extensionService.getExtensionPoint(IReloadGenerator.class); - if (reloadGenerator == null) { - activeHistories = engine.getTriggerRouterService().getActiveTriggerHistories(); - } for (TableReloadRequest load : loadsToProcess) { Node targetNode = engine.getNodeService().findNode(load.getTargetNodeId(), true); @@ -251,12 +246,9 @@ protected void processTableRequestLoads(Node source, ProcessInfo processInfo) { fullLoad.add(load); List triggerRouters = getTriggerRoutersForNodeGroup(triggerRoutersByNodeGroup, targetNode.getNodeGroupId()); - if (reloadGenerator != null) { - activeHistories = reloadGenerator.getActiveTriggerHistories(targetNode); - } - extractRequests = engine.getDataService().insertReloadEvents(targetNode, false, fullLoad, processInfo, activeHistories, - triggerRouters, extractRequests); + extractRequests = engine.getDataService().insertReloadEvents(targetNode, false, fullLoad, processInfo, + triggerRouters, extractRequests, reloadGenerator); loadCountToProcess--; if (++activeLoadCount >= maxLoadCount) { @@ -302,12 +294,9 @@ protected void processTableRequestLoads(Node source, ProcessInfo processInfo) { targetNode.getNodeGroupId()); triggerRoutersByTargetNodeGroupId.put(targetNode.getNodeGroupId(), triggerRouters); } - if (reloadGenerator != null) { - activeHistories = reloadGenerator.getActiveTriggerHistories(targetNode); - } - extractRequests = engine.getDataService().insertReloadEvents(targetNode, false, entry.getValue(), processInfo, activeHistories, - triggerRouters, extractRequests); + extractRequests = engine.getDataService().insertReloadEvents(targetNode, false, entry.getValue(), processInfo, + triggerRouters, extractRequests, reloadGenerator); loadCountToProcess--; if (++activeLoadCount >= maxLoadCount) { @@ -353,23 +342,23 @@ protected List findNodesThatAreReadyForInitialLoad() { return toReturn; } - protected void sendReverseInitialLoad(ProcessInfo processInfo, List activeHistories, - Map> triggerRoutersByNodeGroup) { - INodeService nodeService = engine.getNodeService(); - boolean queuedLoad = false; - List nodes = new ArrayList(); - nodes.addAll(nodeService.findTargetNodesFor(NodeGroupLinkAction.P)); - nodes.addAll(nodeService.findTargetNodesFor(NodeGroupLinkAction.W)); - for (Node node : nodes) { - List triggerRouters = getTriggerRoutersForNodeGroup(triggerRoutersByNodeGroup, node.getNodeGroupId()); - engine.getDataService().insertReloadEvents(node, true, null, processInfo, activeHistories, triggerRouters, null); - queuedLoad = true; - } - - if (!queuedLoad) { - log.info("{} was enabled but no nodes were linked to load", ParameterConstants.AUTO_RELOAD_REVERSE_ENABLED); - } - } +// protected void sendReverseInitialLoad(ProcessInfo processInfo, List activeHistories, +// Map> triggerRoutersByNodeGroup) { +// INodeService nodeService = engine.getNodeService(); +// boolean queuedLoad = false; +// List nodes = new ArrayList(); +// nodes.addAll(nodeService.findTargetNodesFor(NodeGroupLinkAction.P)); +// nodes.addAll(nodeService.findTargetNodesFor(NodeGroupLinkAction.W)); +// for (Node node : nodes) { +// List triggerRouters = getTriggerRoutersForNodeGroup(triggerRoutersByNodeGroup, node.getNodeGroupId()); +// engine.getDataService().insertReloadEvents(node, true, null, processInfo, activeHistories, triggerRouters, null); +// queuedLoad = true; +// } +// +// if (!queuedLoad) { +// log.info("{} was enabled but no nodes were linked to load", ParameterConstants.AUTO_RELOAD_REVERSE_ENABLED); +// } +// } protected boolean isValidLoadTarget(String targetNodeId) { boolean result = false;