From 171b209e80f8c83720e58dde8e919ba7a414dfc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20W=C3=B6hrl?= Date: Thu, 3 May 2018 20:59:10 +0200 Subject: [PATCH 1/8] do not send sync_disabled during registration attempt --- .../symmetric/service/impl/NodeService.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java index 5dd77f4ee9..15f1f8934b 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java @@ -924,7 +924,11 @@ public AuthenticationStatus getAuthenticationStatus(String nodeId, String securi if (node == null) { retVal = AuthenticationStatus.REGISTRATION_REQUIRED; } else if (!syncEnabled(node)) { - retVal = AuthenticationStatus.SYNC_DISABLED; + if(registrationOpen(node)){ + retVal = AuthenticationStatus.REGISTRATION_REQUIRED; + }else{ + retVal = AuthenticationStatus.SYNC_DISABLED; + } } else if (!isNodeAuthorized(nodeId, securityToken)) { retVal = AuthenticationStatus.FORBIDDEN; } @@ -937,6 +941,14 @@ protected boolean syncEnabled(Node node) { syncEnabled = node.isSyncEnabled(); } return syncEnabled; - } + } + + protected boolean registrationOpen(Node node){ + NodeSecurity security = findNodeSecurity(node.getNodeId()); + if(security != null){ + return security.isRegistrationEnabled(); + } + return false; + } } From 29d3a4ef93b93a5ae211cb5658caa37e57d028da Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Wed, 24 Oct 2018 17:10:00 -0400 Subject: [PATCH 2/8] 0003378: SLF4J warnings using symadmin it was either get rid of nuodb from the classpath or back slf4j to version 1.7.6 (same as which was packaged with nuodb). 1.7.6 won (this time). --- symmetric-assemble/common.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-assemble/common.gradle b/symmetric-assemble/common.gradle index c433de3cef..ffefa2be38 100644 --- a/symmetric-assemble/common.gradle +++ b/symmetric-assemble/common.gradle @@ -186,7 +186,7 @@ subprojects { subproject -> jdomVersion = '2.0.5' junitVersion = '4.11' log4jVersion = '1.2.17' - slf4jVersion = '1.7.21' + slf4jVersion = '1.7.6' mockitoVersion = '1.9.5' powerMockVersion = '1.5.3' mysqlVersion = '5.1.45' From de66bae7c4d9a105b2955cbdcf3a72741dc59a3b Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Thu, 25 Oct 2018 12:02:30 -0400 Subject: [PATCH 3/8] 0003768: On SQL Server 2016 all datetime2 columns are altered on every restart --- .../org/jumpmind/db/platform/mssql/MsSqlDdlReader.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mssql/MsSqlDdlReader.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mssql/MsSqlDdlReader.java index 2e9d215097..54c1fa3d4e 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mssql/MsSqlDdlReader.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mssql/MsSqlDdlReader.java @@ -165,7 +165,7 @@ protected Integer mapUnknownJdbcTypeForColumn(Map values) { if (isNotBlank(columnSize)) { size = Integer.parseInt(columnSize); } - if (typeName != null) { + if (typeName != null) { if (typeName.toLowerCase().startsWith("text")) { return Types.LONGVARCHAR; } else if ( typeName.toLowerCase().startsWith("ntext")) { @@ -182,9 +182,11 @@ protected Integer mapUnknownJdbcTypeForColumn(Map values) { return Types.LONGNVARCHAR; } else if ( typeName.toUpperCase().equals("SQL_VARIANT")) { return Types.BINARY; - } else if (typeName != null && typeName.equalsIgnoreCase("DATETIMEOFFSET")) { + } else if (typeName.equalsIgnoreCase("DATETIMEOFFSET")) { return MAPPED_TIMESTAMPTZ; - } + } else if (typeName.equalsIgnoreCase("datetime2")) { + return Types.TIMESTAMP; + } } return super.mapUnknownJdbcTypeForColumn(values); } From 665e1fcd821c2f558b5dbc4f5595045786f749ac Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 26 Oct 2018 12:25:45 -0400 Subject: [PATCH 4/8] 0003764: Loading of Oracle Geography/Geometry Type fails when SRID is explicit on the source column --- .../db/oracle/OracleSymmetricDialect.java | 28 +++++++++---------- .../platform/oracle/OracleDmlStatement.java | 15 ++++++++-- .../org/jumpmind/db/sql/DmlStatement.java | 11 +++++++- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java index 6cab970d6b..a17596f4dc 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleSymmetricDialect.java @@ -191,20 +191,20 @@ public void createRequiredDatabaseObjects() { String wkt2geom = this.parameterService.getTablePrefix() + "_" + "wkt2geom"; if (!installed(SQL_OBJECT_INSTALLED, wkt2geom)) { - String sql = " CREATE OR REPLACE " - + " FUNCTION $(functionName)( " - + " clob_in IN CLOB) " - + " RETURN SDO_GEOMETRY " - + " AS " - + " v_out SDO_GEOMETRY := NULL; " - + " BEGIN " - + " IF clob_in IS NOT NULL THEN " - + " IF DBMS_LOB.GETLENGTH(clob_in) > 0 THEN " - + " v_out := SDO_GEOMETRY(clob_in); " - + " END IF; " - + " END IF; " - + " RETURN v_out; " - + " END $(functionName); "; + String sql = " CREATE OR REPLACE FUNCTION $(functionName) ( \r\n" + + " clob_in IN CLOB, \r\n" + + " srid_in IN INTEGER) \r\n" + + " RETURN SDO_GEOMETRY \r\n" + + " AS \r\n" + + " v_out SDO_GEOMETRY := NULL; \r\n" + + " BEGIN \r\n" + + " IF clob_in IS NOT NULL THEN \r\n" + + " IF DBMS_LOB.GETLENGTH(clob_in) > 0 THEN \r\n" + + " v_out := SDO_GEOMETRY(clob_in, srid_in); \r\n" + + " END IF; \r\n" + + " END IF; \r\n" + + " RETURN v_out; \r\n" + + " END $(functionName); \r\n"; install(sql, wkt2geom); } diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java index 2a28b78ab7..980b09c281 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java @@ -22,6 +22,7 @@ import java.sql.Types; +import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Column; import org.jumpmind.db.model.TypeMap; import org.jumpmind.db.platform.DatabaseInfo; @@ -42,7 +43,7 @@ protected void appendColumnParameter(StringBuilder sql, Column column) { sql.append("TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')") .append(","); } else if (isGeometry(column)) { - sql.append("SYM_WKT2GEOM(?)").append(","); + sql.append("SYM_WKT2GEOM(?,").append(buildSRIDSelect(column)).append(")").append(","); } else if (column.getJdbcTypeName().startsWith("XMLTYPE")) { sql.append("XMLTYPE(?)").append(","); } else { @@ -57,7 +58,7 @@ protected void appendColumnEquals(StringBuilder sql, Column column) { .append(" = TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')"); } else if (isGeometry(column)) { sql.append(quote).append(column.getName()).append(quote).append(" = ") - .append("SYM_WKT2GEOM(?)"); + .append("SYM_WKT2GEOM(?,").append(buildSRIDSelect(column)).append(")"); } else if (column.getJdbcTypeName().startsWith("XMLTYPE")) { sql.append(quote).append(column.getName()).append(quote).append(" = ") .append("XMLTYPE(?)"); @@ -97,4 +98,14 @@ protected boolean isGeometry(Column column) { return false; } } + + protected String buildSRIDSelect(Column column) { + if (!StringUtils.isEmpty(schemaName)) { + return String.format("(select SRID from all_sdo_geom_metadata where owner = '%s' and table_name = '%s' and column_name = '%s')", + schemaName.toUpperCase(), tableName.toUpperCase(), column.getName().toUpperCase()); + } else { + return String.format("(select SRID from user_sdo_geom_metadata where table_name = '%s' and column_name = '%s')", + tableName.toUpperCase(), column.getName().toUpperCase()); + } + } } diff --git a/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java index f55705ea00..b09db0b2f0 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java @@ -53,6 +53,12 @@ public class DmlStatement { public enum DmlType { INSERT, UPDATE, DELETE, UPSERT, COUNT, FROM, WHERE, SELECT, SELECT_ALL, UNKNOWN }; + + protected String catalogName; + + protected String schemaName; + + protected String tableName; protected DmlType dmlType; @@ -99,7 +105,10 @@ protected void init(DmlType type, String catalogName, String schemaName, String Column[] keysColumns, Column[] columns, boolean[] nullKeyValues, DatabaseInfo databaseInfo, boolean useQuotedIdentifiers, String textColumnExpression, boolean namedParameters) { - + + this.catalogName = catalogName; + this.schemaName = schemaName; + this.tableName = tableName; this.namedParameters = namedParameters; this.databaseInfo = databaseInfo; this.columns = columns; From b9ec80b9c637630f1901e2efad9f03979b86795d Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 26 Oct 2018 12:28:15 -0400 Subject: [PATCH 5/8] 0003765: If Sym_table_reload_request's router_id is invalid, a full table load occurs (any 'where' clause is ignored) --- .../symmetric/service/impl/DataService.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java index f732f1c3d5..15da2ab67d 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java @@ -38,6 +38,7 @@ import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Column; @@ -464,7 +465,7 @@ public void insertReloadEvents(Node targetNode, boolean reverse, List mapReloadRequests = convertReloadListToMap(reloadRequests); + Map mapReloadRequests = convertReloadListToMap(reloadRequests, triggerRouters); String symNodeSecurityReloadChannel = null; int totalTableCount = 0; @@ -585,17 +586,35 @@ private String findChannelFor(TriggerHistory history, List trigge } @SuppressWarnings("unchecked") - protected Map convertReloadListToMap(List reloadRequests) { + protected Map convertReloadListToMap(List reloadRequests, List triggerRouters) { if (reloadRequests == null) { return null; } Map reloadMap = new CaseInsensitiveMap(); - for (TableReloadRequest item : reloadRequests) { - reloadMap.put(item.getIdentifier(), item); + for (TableReloadRequest reloadRequest : reloadRequests) { + validate(reloadRequest, triggerRouters); + reloadMap.put(reloadRequest.getIdentifier(), reloadRequest); } return reloadMap; } + protected void validate(TableReloadRequest reloadRequest, List triggerRouters) { + boolean validMatch = false; + for (TriggerRouter triggerRouter : triggerRouters) { + if (ObjectUtils.equals(triggerRouter.getTriggerId(), reloadRequest.getTriggerId()) + && ObjectUtils.equals(triggerRouter.getRouterId(), reloadRequest.getRouterId())) { + validMatch = true; + break; + } + } + + if (!validMatch) { + throw new SymmetricException("Table reload request submitted which does not have a valid trigger/router " + + "combination in sym_trigger_router. Request trigger id: '" + reloadRequest.getTriggerId() + "' router id: '" + + reloadRequest.getRouterId() + "' create time: " + reloadRequest.getCreateTime()); + } + } + private void callReloadListeners(boolean before, Node targetNode, boolean transactional, ISqlTransaction transaction, long loadId) { for (IReloadListener listener : extensionService.getExtensionPointList(IReloadListener.class)) { From 17379136cae66badb699b72e9e552427eac29c48 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 26 Oct 2018 12:45:07 -0400 Subject: [PATCH 6/8] 0003747: Symmetric should log full SQL Statement when conflict resolution SQL fails --- .../DefaultDatabaseWriterConflictResolver.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java index c260bba29a..eeaf532c66 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriterConflictResolver.java @@ -22,6 +22,7 @@ import java.sql.Timestamp; import java.sql.Types; +import java.util.Arrays; import java.util.Date; import java.util.Map; import java.util.TimeZone; @@ -118,8 +119,15 @@ protected boolean isVersionNewer(Conflict conflict, AbstractDatabaseWriter write DmlStatement stmt = databaseWriter.getPlatform().createDmlStatement(DmlType.FROM, targetTable , writer.getWriterSettings().getTextColumnExpression()); String sql = stmt.getColumnsSql(new Column[] { targetTable.getColumnWithName(columnName) }); - Long existingVersion = databaseWriter.getTransaction() - .queryForObject(sql, Long.class, objectValues); + Long existingVersion = null; + + try { + existingVersion = databaseWriter.getTransaction().queryForObject(sql, Long.class, objectValues); + } catch (Exception ex) { + throw new RuntimeException("Failed to execute conflict resolution SQL: \"" + + sql + "\" values: " + Arrays.toString(objectValues), ex); + } + if (existingVersion == null) { return true; } else { From bf744546832ec8b7407b185d33afb3609daddfcd Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Fri, 26 Oct 2018 12:48:41 -0400 Subject: [PATCH 7/8] 0003762: Bulk loader fallback to default loader and fails will mark batch ok on source when it did not load the batch. --- .../org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java | 3 +++ .../org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java index 0516cf659d..ccb5c7b671 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/AbstractBulkDatabaseWriter.java @@ -1,6 +1,8 @@ package org.jumpmind.symmetric.io; import org.jumpmind.db.platform.IDatabasePlatform; +import org.jumpmind.db.sql.JdbcSqlTemplate; +import org.jumpmind.db.sql.JdbcSqlTransaction; import org.jumpmind.symmetric.common.ContextConstants; import org.jumpmind.symmetric.io.data.Batch; import org.jumpmind.symmetric.io.data.CsvData; @@ -22,6 +24,7 @@ public AbstractBulkDatabaseWriter(IDatabasePlatform symmetricPlatform, IDatabase public void start(Batch batch) { super.start(batch); if (isFallBackToDefault()) { + getTransaction().setInBatchMode(false); log.debug("Writing batch " + batch.getBatchId() + " on channel " + batch.getChannelId() + " to node " + batch.getTargetNodeId() + " using DEFAULT loader"); }else{ log.debug("Writing batch " + batch.getBatchId() + " on channel " + batch.getChannelId() + " to node " + batch.getTargetNodeId() + " using BULK loader"); diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java index ae98bf7c23..28e63a72a1 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/JdbcBatchBulkDatabaseWriter.java @@ -34,7 +34,7 @@ protected LoadStatus insert(CsvData data) { @Override protected LoadStatus update(CsvData data, boolean applyChangesOnly, boolean useConflictDetection) { - LoadStatus loadStatus = super.insert(data); + LoadStatus loadStatus = super.update(data, applyChangesOnly, useConflictDetection); if (loadStatus == LoadStatus.CONFLICT) { loadStatus = LoadStatus.SUCCESS; } From 4da2a5deb9190aab547b409770c6f240ee0a3aad Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 29 Oct 2018 15:20:06 -0400 Subject: [PATCH 8/8] 0003540: Registration gets lost during the registration when File Sync is enabled --- .../org/jumpmind/symmetric/service/impl/FileSyncService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java index 765fe67231..6ac57174a9 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java @@ -146,6 +146,10 @@ public void trackChanges(boolean force) { try { log.debug("Tracking changes for file sync"); Node local = engine.getNodeService().findIdentity(); + if (local == null) { + log.warn("Not running file sync trackChanges because the local node is not available yet. It may not be registered yet."); + return; + } ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo( new ProcessInfoKey(local.getNodeId(), null, ProcessType.FILE_SYNC_TRACKER)); boolean useCrc = engine.getParameterService().is(ParameterConstants.FILE_SYNC_USE_CRC);