From 171b209e80f8c83720e58dde8e919ba7a414dfc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20W=C3=B6hrl?= Date: Thu, 3 May 2018 20:59:10 +0200 Subject: [PATCH 01/15] do not send sync_disabled during registration attempt --- .../symmetric/service/impl/NodeService.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java index 5dd77f4ee9..15f1f8934b 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java @@ -924,7 +924,11 @@ public AuthenticationStatus getAuthenticationStatus(String nodeId, String securi if (node == null) { retVal = AuthenticationStatus.REGISTRATION_REQUIRED; } else if (!syncEnabled(node)) { - retVal = AuthenticationStatus.SYNC_DISABLED; + if(registrationOpen(node)){ + retVal = AuthenticationStatus.REGISTRATION_REQUIRED; + }else{ + retVal = AuthenticationStatus.SYNC_DISABLED; + } } else if (!isNodeAuthorized(nodeId, securityToken)) { retVal = AuthenticationStatus.FORBIDDEN; } @@ -937,6 +941,14 @@ protected boolean syncEnabled(Node node) { syncEnabled = node.isSyncEnabled(); } return syncEnabled; - } + } + + protected boolean registrationOpen(Node node){ + NodeSecurity security = findNodeSecurity(node.getNodeId()); + if(security != null){ + return security.isRegistrationEnabled(); + } + return false; + } } From 665e1fcd821c2f558b5dbc4f5595045786f749ac Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 26 Oct 2018 12:25:45 -0400 Subject: [PATCH 02/15] 0003764: Loading of Oracle Geography/Geometry Type fails when SRID is explicit on the source column --- .../db/oracle/OracleSymmetricDialect.java | 28 +++++++++---------- .../platform/oracle/OracleDmlStatement.java | 15 ++++++++-- .../org/jumpmind/db/sql/DmlStatement.java | 11 +++++++- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java index 6cab970d6b..a17596f4dc 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java @@ -191,20 +191,20 @@ public void createRequiredDatabaseObjects() { String wkt2geom = this.parameterService.getTablePrefix() + "_" + "wkt2geom"; if (!installed(SQL_OBJECT_INSTALLED, wkt2geom)) { - String sql = " CREATE OR REPLACE " - + " FUNCTION $(functionName)( " - + " clob_in IN CLOB) " - + " RETURN SDO_GEOMETRY " - + " AS " - + " v_out SDO_GEOMETRY := NULL; " - + " BEGIN " - + " IF clob_in IS NOT NULL THEN " - + " IF DBMS_LOB.GETLENGTH(clob_in) > 0 THEN " - + " v_out := SDO_GEOMETRY(clob_in); " - + " END IF; " - + " END IF; " - + " RETURN v_out; " - + " END $(functionName); "; + String sql = " CREATE OR REPLACE FUNCTION $(functionName) ( \r\n" + + " clob_in IN CLOB, \r\n" + + " srid_in IN INTEGER) \r\n" + + " RETURN SDO_GEOMETRY \r\n" + + " AS \r\n" + + " v_out SDO_GEOMETRY := NULL; \r\n" + + " BEGIN \r\n" + + " IF clob_in IS NOT NULL THEN \r\n" + + " IF DBMS_LOB.GETLENGTH(clob_in) > 0 THEN \r\n" + + " v_out := SDO_GEOMETRY(clob_in, srid_in); \r\n" + + " END IF; \r\n" + + " END IF; \r\n" + + " RETURN v_out; \r\n" + + " END $(functionName); \r\n"; install(sql, wkt2geom); } diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java index 2a28b78ab7..980b09c281 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java @@ -22,6 +22,7 @@ import java.sql.Types; +import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Column; import org.jumpmind.db.model.TypeMap; import org.jumpmind.db.platform.DatabaseInfo; @@ -42,7 +43,7 @@ protected void appendColumnParameter(StringBuilder sql, Column column) { sql.append("TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')") .append(","); } else if (isGeometry(column)) { - sql.append("SYM_WKT2GEOM(?)").append(","); + sql.append("SYM_WKT2GEOM(?,").append(buildSRIDSelect(column)).append(")").append(","); } else if (column.getJdbcTypeName().startsWith("XMLTYPE")) { sql.append("XMLTYPE(?)").append(","); } else { @@ -57,7 +58,7 @@ protected void appendColumnEquals(StringBuilder sql, Column column) { .append(" = TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')"); } else if (isGeometry(column)) { sql.append(quote).append(column.getName()).append(quote).append(" = ") - .append("SYM_WKT2GEOM(?)"); + .append("SYM_WKT2GEOM(?,").append(buildSRIDSelect(column)).append(")"); } else if (column.getJdbcTypeName().startsWith("XMLTYPE")) { sql.append(quote).append(column.getName()).append(quote).append(" = ") .append("XMLTYPE(?)"); @@ -97,4 +98,14 @@ protected boolean isGeometry(Column column) { return false; } } + + protected String buildSRIDSelect(Column column) { + if (!StringUtils.isEmpty(schemaName)) { + return String.format("(select SRID from all_sdo_geom_metadata where owner = '%s' and table_name = '%s' and column_name = '%s')", + schemaName.toUpperCase(), tableName.toUpperCase(), column.getName().toUpperCase()); + } else { + return String.format("(select SRID from user_sdo_geom_metadata where table_name = '%s' and column_name = '%s')", + tableName.toUpperCase(), column.getName().toUpperCase()); + } + } } diff --git a/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java index f55705ea00..b09db0b2f0 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java @@ -53,6 +53,12 @@ public class DmlStatement { public enum DmlType { INSERT, UPDATE, DELETE, UPSERT, COUNT, FROM, WHERE, SELECT, SELECT_ALL, UNKNOWN }; + + protected String catalogName; + + protected String schemaName; + + protected String tableName; protected DmlType dmlType; @@ -99,7 +105,10 @@ protected void init(DmlType type, String catalogName, String schemaName, String Column[] keysColumns, Column[] columns, boolean[] nullKeyValues, DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression, boolean namedParameters) { - + + this.catalogName = catalogName; + this.schemaName = schemaName; + this.tableName = tableName; this.namedParameters = namedParameters; this.databaseInfo = databaseInfo; this.columns = columns; From b9ec80b9c637630f1901e2efad9f03979b86795d Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 26 Oct 2018 12:28:15 -0400 Subject: [PATCH 03/15] 0003765: If Sym_table_reload_request's router_id is invalid, a full table load occurs (any 'where' clause is ignored) --- .../symmetric/service/impl/DataService.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java index f732f1c3d5..15da2ab67d 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java @@ -38,6 +38,7 @@ import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Column; @@ -464,7 +465,7 @@ public void insertReloadEvents(Node targetNode, boolean reverse, List mapReloadRequests = convertReloadListToMap(reloadRequests); + Map mapReloadRequests = convertReloadListToMap(reloadRequests, triggerRouters); String symNodeSecurityReloadChannel = null; int totalTableCount = 0; @@ -585,17 +586,35 @@ private String findChannelFor(TriggerHistory history, List trigge } @SuppressWarnings("unchecked") - protected Map convertReloadListToMap(List reloadRequests) { + protected Map convertReloadListToMap(List reloadRequests, List triggerRouters) { if (reloadRequests == null) { return null; } Map reloadMap = new CaseInsensitiveMap(); - for (TableReloadRequest item : reloadRequests) { - reloadMap.put(item.getIdentifier(), item); + for (TableReloadRequest reloadRequest : reloadRequests) { + validate(reloadRequest, triggerRouters); + reloadMap.put(reloadRequest.getIdentifier(), reloadRequest); } return reloadMap; } + protected void validate(TableReloadRequest reloadRequest, List triggerRouters) { + boolean validMatch = false; + for (TriggerRouter triggerRouter : triggerRouters) { + if (ObjectUtils.equals(triggerRouter.getTriggerId(), reloadRequest.getTriggerId()) + && ObjectUtils.equals(triggerRouter.getRouterId(), reloadRequest.getRouterId())) { + validMatch = true; + break; + } + } + + if (!validMatch) { + throw new SymmetricException("Table reload request submitted which does not have a valid trigger/router " + + "combination in sym_trigger_router. Request trigger id: '" + reloadRequest.getTriggerId() + "' router id: '" + + reloadRequest.getRouterId() + "' create time: " + reloadRequest.getCreateTime()); + } + } + private void callReloadListeners(boolean before, Node targetNode, boolean transactional, ISqlTransaction transaction, long loadId) { for (IReloadListener listener : extensionService.getExtensionPointList(IReloadListener.class)) { From 17379136cae66badb699b72e9e552427eac29c48 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 26 Oct 2018 12:45:07 -0400 Subject: [PATCH 04/15] 0003747: Symmetric should log full SQL Statement when conflict resolution SQL fails --- .../DefaultDatabaseWriterConflictResolver.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java index c260bba29a..eeaf532c66 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java @@ -22,6 +22,7 @@ import java.sql.Timestamp; import java.sql.Types; +import java.util.Arrays; import java.util.Date; import java.util.Map; import java.util.TimeZone; @@ -118,8 +119,15 @@ protected boolean isVersionNewer(Conflict conflict, AbstractDatabaseWriter write DmlStatement stmt = databaseWriter.getPlatform().createDmlStatement(DmlType.FROM, targetTable , writer.getWriterSettings().getTextColumnExpression()); String sql = stmt.getColumnsSql(new Column[] { targetTable.getColumnWithName(columnName) }); - Long existingVersion = databaseWriter.getTransaction() - .queryForObject(sql, Long.class, objectValues); + Long existingVersion = null; + + try { + existingVersion = databaseWriter.getTransaction().queryForObject(sql, Long.class, objectValues); + } catch (Exception ex) { + throw new RuntimeException("Failed to execute conflict resolution SQL: \"" + + sql + "\" values: " + Arrays.toString(objectValues), ex); + } + if (existingVersion == null) { return true; } else { From bf744546832ec8b7407b185d33afb3609daddfcd Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Fri, 26 Oct 2018 12:48:41 -0400 Subject: [PATCH 05/15] 0003762: Bulk loader fallback to default loader and fails will mark batch ok on source when it did not load the batch. --- .../org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java | 3 +++ .../org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java index 0516cf659d..ccb5c7b671 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java @@ -1,6 +1,8 @@ package org.jumpmind.symmetric.io; import org.jumpmind.db.platform.IDatabasePlatform; +import org.jumpmind.db.sql.JdbcSqlTemplate; +import org.jumpmind.db.sql.JdbcSqlTransaction; import org.jumpmind.symmetric.common.ContextConstants; import org.jumpmind.symmetric.io.data.Batch; import org.jumpmind.symmetric.io.data.CsvData; @@ -22,6 +24,7 @@ public AbstractBulkDatabaseWriter(IDatabasePlatform symmetricPlatform, IDatabase public void start(Batch batch) { super.start(batch); if (isFallBackToDefault()) { + getTransaction().setInBatchMode(false); log.debug("Writing batch " + batch.getBatchId() + " on channel " + batch.getChannelId() + " to node " + batch.getTargetNodeId() + " using DEFAULT loader"); }else{ log.debug("Writing batch " + batch.getBatchId() + " on channel " + batch.getChannelId() + " to node " + batch.getTargetNodeId() + " using BULK loader"); diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java index ae98bf7c23..28e63a72a1 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java @@ -34,7 +34,7 @@ protected LoadStatus insert(CsvData data) { @Override protected LoadStatus update(CsvData data, boolean applyChangesOnly, boolean useConflictDetection) { - LoadStatus loadStatus = super.insert(data); + LoadStatus loadStatus = super.update(data, applyChangesOnly, useConflictDetection); if (loadStatus == LoadStatus.CONFLICT) { loadStatus = LoadStatus.SUCCESS; } From ed03f8fcdbdfb28582592925410891d001b51264 Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Mon, 29 Oct 2018 12:53:18 -0400 Subject: [PATCH 06/15] 0003765: If Sym_table_reload_request's router_id is invalid, a full table load occurs (any 'where' clause is ignored) --- .../org/jumpmind/symmetric/service/impl/DataService.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java index 15da2ab67d..9c4e3e8317 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java @@ -465,7 +465,7 @@ public void insertReloadEvents(Node targetNode, boolean reverse, List mapReloadRequests = convertReloadListToMap(reloadRequests, triggerRouters); + Map mapReloadRequests = convertReloadListToMap(reloadRequests, triggerRouters, isFullLoad); String symNodeSecurityReloadChannel = null; int totalTableCount = 0; @@ -586,13 +586,15 @@ private String findChannelFor(TriggerHistory history, List trigge } @SuppressWarnings("unchecked") - protected Map convertReloadListToMap(List reloadRequests, List triggerRouters) { + protected Map convertReloadListToMap(List reloadRequests, List triggerRouters, boolean isFullLoad) { if (reloadRequests == null) { return null; } Map reloadMap = new CaseInsensitiveMap(); for (TableReloadRequest reloadRequest : reloadRequests) { - validate(reloadRequest, triggerRouters); + if (!isFullLoad) { + validate(reloadRequest, triggerRouters); + } reloadMap.put(reloadRequest.getIdentifier(), reloadRequest); } return reloadMap; From 29dc37749de8d450ef69d4ee7f48061372a82520 Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Mon, 29 Oct 2018 12:58:10 -0400 Subject: [PATCH 07/15] 0003767: Kafka performance improvements --- .../symmetric/load/KafkaWriterFilter.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/load/KafkaWriterFilter.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/load/KafkaWriterFilter.java index 736e918ff2..7c11394328 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/load/KafkaWriterFilter.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/load/KafkaWriterFilter.java @@ -231,7 +231,7 @@ else if (Long.class.equals(propertyTypeClass)) { } } } - sendKafkaMessage(pojo, kafkaDataKey); + sendKafkaMessageByObject(pojo, kafkaDataKey); } else { throw new RuntimeException("Unable to find a POJO to load for AVRO based message onto Kafka for table : " + tableName); } @@ -396,19 +396,23 @@ public void earlyCommit(DataContext context) { public void batchComplete(DataContext context) { if (!context.getBatch().getChannelId().equals("heartbeat") && !context.getBatch().getChannelId().equals("config")) { String batchFileName = "batch-" + context.getBatch().getSourceNodeId() + "-" + context.getBatch().getBatchId(); + + KafkaProducer producer = new KafkaProducer(configs); try { if (confluentUrl == null && kafkaDataMap.size() > 0) { StringBuffer kafkaText = new StringBuffer(); + + for (Map.Entry> entry : kafkaDataMap.entrySet()) { for (String row : entry.getValue()) { if (messageBy.equals(KAFKA_MESSAGE_BY_ROW)) { - sendKafkaMessage(row, entry.getKey()); + sendKafkaMessage(producer, row, entry.getKey()); } else { kafkaText.append(row); } } if (messageBy.equals(KAFKA_MESSAGE_BY_BATCH)) { - sendKafkaMessage(kafkaText.toString(), entry.getKey()); + sendKafkaMessage(producer, kafkaText.toString(), entry.getKey()); } } kafkaDataMap = new HashMap>(); @@ -417,6 +421,7 @@ public void batchComplete(DataContext context) { log.warn("Unable to write batch to Kafka " + batchFileName, e); e.printStackTrace(); } finally { + producer.close(); context.put(KAFKA_TEXT_CACHE, new HashMap>()); tableNameCache.clear(); tableColumnCache = new HashMap>(); @@ -430,15 +435,12 @@ public void batchCommitted(DataContext context) { public void batchRolledback(DataContext context) { } - public void sendKafkaMessage(String kafkaText, String topic) { - KafkaProducer producer = new KafkaProducer(configs); + public void sendKafkaMessage(KafkaProducer producer, String kafkaText, String topic) { producer.send(new ProducerRecord(topic, kafkaText)); log.debug("Data to be sent to Kafka-" + kafkaText); - - producer.close(); } - public void sendKafkaMessage(Object bean, String topic) { + public void sendKafkaMessageByObject(Object bean, String topic) { KafkaProducer producer = new KafkaProducer(configs); producer.send(new ProducerRecord(topic, bean)); producer.close(); From 4da2a5deb9190aab547b409770c6f240ee0a3aad Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 15:20:06 -0400 Subject: [PATCH 08/15] 0003540: Registration gets lost during the registration when File Sync is enabled --- .../org/jumpmind/symmetric/service/impl/FileSyncService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java index 765fe67231..6ac57174a9 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java @@ -146,6 +146,10 @@ public void trackChanges(boolean force) { try { log.debug("Tracking changes for file sync"); Node local = engine.getNodeService().findIdentity(); + if (local == null) { + log.warn("Not running file sync trackChanges because the local node is not available yet. It may not be registered yet."); + return; + } ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo( new ProcessInfoKey(local.getNodeId(), null, ProcessType.FILE_SYNC_TRACKER)); boolean useCrc = engine.getParameterService().is(ParameterConstants.FILE_SYNC_USE_CRC); From 3b03c857e86c45b7529526192ade069768995d83 Mon Sep 17 00:00:00 2001 From: elong Date: Mon, 29 Oct 2018 15:35:04 -0400 Subject: [PATCH 09/15] 0003772: Cannot inactive table for sync when one of triggers is missing --- .../service/impl/TriggerRouterService.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java index 721020a24e..c032b361ff 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java @@ -1343,20 +1343,14 @@ public void dropTriggers(TriggerHistory history) { protected void dropTriggers(TriggerHistory history, StringBuilder sqlBuffer) { try { - if (StringUtils.isNotBlank(history.getNameForInsertTrigger())) { - symmetricDialect.removeTrigger(sqlBuffer, history.getSourceCatalogName(), history.getSourceSchemaName(), - history.getNameForInsertTrigger(), history.getSourceTableName()); - } + dropTrigger(sqlBuffer, history.getSourceCatalogName(), history.getSourceSchemaName(), + history.getNameForInsertTrigger(), history.getSourceTableName()); - if (StringUtils.isNotBlank(history.getNameForDeleteTrigger())) { - symmetricDialect.removeTrigger(sqlBuffer, history.getSourceCatalogName(), history.getSourceSchemaName(), - history.getNameForDeleteTrigger(), history.getSourceTableName()); - } + dropTrigger(sqlBuffer, history.getSourceCatalogName(), history.getSourceSchemaName(), + history.getNameForDeleteTrigger(), history.getSourceTableName()); - if (StringUtils.isNotBlank(history.getNameForUpdateTrigger())) { - symmetricDialect.removeTrigger(sqlBuffer, history.getSourceCatalogName(), history.getSourceSchemaName(), - history.getNameForUpdateTrigger(), history.getSourceTableName()); - } + dropTrigger(sqlBuffer, history.getSourceCatalogName(), history.getSourceSchemaName(), + history.getNameForUpdateTrigger(), history.getSourceTableName()); if (parameterService.is(ParameterConstants.AUTO_SYNC_TRIGGERS)) { for (ITriggerCreationListener l : extensionService.getExtensionPointList(ITriggerCreationListener.class)) { @@ -1381,6 +1375,16 @@ protected void dropTriggers(TriggerHistory history, StringBuilder sqlBuffer) { log.error("Error while dropping triggers for table {}", history.getSourceTableName(), ex); } } + + protected void dropTrigger(StringBuilder sqlBuffer, String catalog, String schema, String triggerName, String tableName) { + if (StringUtils.isNotBlank(triggerName)) { + try { + symmetricDialect.removeTrigger(sqlBuffer, catalog, schema, triggerName, tableName); + } catch (Throwable e) { + log.error("Error while dropping trigger {} for table {}", triggerName, tableName, e); + } + } + } protected List toList(Collection> source) { ArrayList list = new ArrayList(); From 8d1280742721eadab50da9291d2fbcd2511e0f7d Mon Sep 17 00:00:00 2001 From: elong Date: Mon, 29 Oct 2018 16:16:51 -0400 Subject: [PATCH 10/15] 0003773: With wildcards, rebuilding specific table trigger can result in multiple active triggers --- .../jumpmind/symmetric/service/impl/TriggerRouterService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java index c032b361ff..95cf1fa689 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java @@ -1511,8 +1511,8 @@ public void syncTriggers(Table table, boolean force) { List triggersForCurrentNode = getTriggersForCurrentNode(); List activeTriggerHistories = getActiveTriggerHistories(); for (Trigger trigger : triggersForCurrentNode) { - if (trigger.matches(table, platform.getDefaultCatalog(), platform.getDefaultSchema(), - ignoreCase)) { + if (trigger.matches(table, platform.getDefaultCatalog(), platform.getDefaultSchema(), ignoreCase) && + (!trigger.isSourceTableNameWildCarded() || !containsExactMatchForSourceTableName(table, triggersForCurrentNode, ignoreCase))) { log.info("Synchronizing triggers for {}", table.getFullyQualifiedTableName()); updateOrCreateDatabaseTriggers(trigger, table, null, force, true, activeTriggerHistories); log.info("Done synchronizing triggers for {}", table.getFullyQualifiedTableName()); From 290095f7c25ccd2f8d06af22239c2a4bdd7c2ad1 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 16:21:24 -0400 Subject: [PATCH 11/15] 0003727: MySQL documentation needs updated to reflect need to grant the PROCESS priv for transaction id --- symmetric-assemble/src/asciidoc/appendix/mysql.ad | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/symmetric-assemble/src/asciidoc/appendix/mysql.ad b/symmetric-assemble/src/asciidoc/appendix/mysql.ad index a1e36e4fd0..88a425d3c7 100644 --- a/symmetric-assemble/src/asciidoc/appendix/mysql.ad +++ b/symmetric-assemble/src/asciidoc/appendix/mysql.ad @@ -31,6 +31,15 @@ grant create routine on *.* to symmetric; grant process on *.* to symmetric; ---- +Starting in MySQL 5.7.6, the "PROCESS" privilege is also required for the MySQL user that is modifying the application tables. +This is required to look up the transaction id. Internally, the trigger will submit this query during an insert/update/delete: + +select TRX_ID from INFORMATION_SCHEMA.INNODB_TRX where TRX_MYSQL_THREAD_ID = CONNECTION_ID(); + +---- +grant process on *.* to db_user; +---- + MySQL allows '0000-00-00 00:00:00' to be entered as a value for datetime and timestamp columns. JDBC cannot deal with a date value with a year of 0. In order to work around this SymmetricDS can be configured to treat date and time columns as varchar columns for data capture and data load. To enable this feature set the db.treat.date.time.as.varchar.enabled property to true. From 69f4ce8dcd6675a66c628a22cd8dcf06f5f0d488 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 16:33:08 -0400 Subject: [PATCH 12/15] 0003622: Monitoring of Heap space memory percent is not firing as expected --- .../java/org/jumpmind/symmetric/monitor/MonitorTypeMemory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeMemory.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeMemory.java index 527f9e2017..4d85723b5a 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeMemory.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/monitor/MonitorTypeMemory.java @@ -61,7 +61,7 @@ public MonitorEvent check(Monitor monitor) { MonitorEvent event = new MonitorEvent(); long usage = 0; if (tenuredPool != null) { - usage = (long) (tenuredPool.getUsage().getUsed() / tenuredPool.getUsage().getMax()); + usage = (long) ((double)tenuredPool.getUsage().getUsed() / (double)tenuredPool.getUsage().getMax() * 100); } event.setValue(usage); return event; From 8f7913cc4de1913dfae55f7127d2384e75fc0ba3 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 16:38:30 -0400 Subject: [PATCH 13/15] 0003527: The jackson databind library has a vulnerability. --- symmetric-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-core/build.gradle b/symmetric-core/build.gradle index e4d2cc8f77..9527a9774c 100644 --- a/symmetric-core/build.gradle +++ b/symmetric-core/build.gradle @@ -7,7 +7,7 @@ apply from: symAssembleDir + '/common.gradle' compile project(":symmetric-util") compile "commons-fileupload:commons-fileupload:$commonsFileuploadVersion" compile "javax.mail:mail:1.4.5" - compile "com.fasterxml.jackson.core:jackson-databind:2.9.3" + compile "com.fasterxml.jackson.core:jackson-databind:2.9.5" compile "com.google.code.gson:gson:2.8.2" compile "org.springframework:spring-core:$springVersion" From 30d07abda20251e6533fd45f484125db56a74a51 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 17:06:23 -0400 Subject: [PATCH 14/15] 0003774: Update gradle to support building with Java 11 --- symmetric-assemble/gradle/wrapper/gradle-wrapper.properties | 2 +- symmetric-sqlexplorer/build.gradle | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/symmetric-assemble/gradle/wrapper/gradle-wrapper.properties b/symmetric-assemble/gradle/wrapper/gradle-wrapper.properties index 5d7992c702..d9bb753fa9 100644 --- a/symmetric-assemble/gradle/wrapper/gradle-wrapper.properties +++ b/symmetric-assemble/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip diff --git a/symmetric-sqlexplorer/build.gradle b/symmetric-sqlexplorer/build.gradle index e29d6c3386..9ef3a41d04 100644 --- a/symmetric-sqlexplorer/build.gradle +++ b/symmetric-sqlexplorer/build.gradle @@ -38,7 +38,9 @@ artifacts { } configurations.archives.with { - artifacts.remove artifacts.find { it.archiveTask.is war } + + + artifacts.remove artifacts.find { it.type == 'war' } } dependencies { From 40b05f845156e4b889d21e2c76bf84211972fbdc Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 17:08:01 -0400 Subject: [PATCH 15/15] 0003526: The spring core library has a vulnerability. --- symmetric-assemble/common.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-assemble/common.gradle b/symmetric-assemble/common.gradle index 7cd6405b48..1f33597a4e 100644 --- a/symmetric-assemble/common.gradle +++ b/symmetric-assemble/common.gradle @@ -191,7 +191,7 @@ subprojects { subproject -> powerMockVersion = '1.5.3' mysqlVersion = '5.1.45' servletVersion = '3.1.0' - springVersion = '4.3.13.RELEASE' + springVersion = '4.3.16.RELEASE' jtdsVersion = '1.2.8' voltDbVersion = '6.2' bouncyCastleVersion = '1.59'