From 2c9ff2ea453178f8727675313755a2664fc718dc Mon Sep 17 00:00:00 2001 From: maxwellpettit Date: Thu, 20 Apr 2017 14:16:53 -0400 Subject: [PATCH 1/8] 0003065: DB2 Error: SQLCODE=-574 during Initial Load when default datatype is CURRENT TIMESTAMP --- .../java/org/jumpmind/db/platform/AbstractDdlBuilder.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDdlBuilder.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDdlBuilder.java index 229730f386..17c5ce09ae 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDdlBuilder.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDdlBuilder.java @@ -2045,7 +2045,11 @@ protected void printDefaultValue(String defaultValue, int typeCode, StringBuilde || defaultValueStr.toUpperCase().startsWith("CURRENT_TIMESTAMP") || defaultValueStr.toUpperCase().startsWith("CURRENT_TIME") || defaultValueStr.toUpperCase().startsWith("CURRENT_DATE") + || defaultValueStr.toUpperCase().startsWith("CURRENT TIMESTAMP") + || defaultValueStr.toUpperCase().startsWith("CURRENT TIME") + || defaultValueStr.toUpperCase().startsWith("CURRENT DATE") || defaultValueStr.toUpperCase().startsWith("CURRENT_USER") + || defaultValueStr.toUpperCase().startsWith("CURRENT USER") || defaultValueStr.toUpperCase().startsWith("USER") || defaultValueStr.toUpperCase().startsWith("SYSTEM_USER") || defaultValueStr.toUpperCase().startsWith("SESSION_USER") From a43bf93508a2c3a46ec85d6872412ac4932d5cf7 Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Thu, 20 Apr 2017 16:45:26 -0400 Subject: [PATCH 2/8] fix sqlite unit test --- .../test/java/org/jumpmind/symmetric/DbExportImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java b/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java index dc51d2fc9d..60922f0271 100644 --- a/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java +++ b/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java @@ -152,7 +152,7 @@ public void exportTestDatabaseSQL() throws Exception { return; } - final int EXPECTED_VARCHAR_MAX_COUNT = engine.getDatabasePlatform().getName().equals(DatabaseNamesConstants.SQLITE) ? 293 : 50; + final int EXPECTED_VARCHAR_MAX_COUNT = engine.getDatabasePlatform().getName().equals(DatabaseNamesConstants.SQLITE) ? 294 : 50; final String EXPECTED_VARCHAR_MAX_STRING = "varchar(" + Integer.MAX_VALUE + ")"; final int actualVarcharMaxCount = StringUtils.countMatches(output, EXPECTED_VARCHAR_MAX_STRING); String msg = String.format("Expected %s, but got %s in the following output %s", From 9cc3f917e78dccda19de78b74ffc6abaa15e0e95 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Fri, 21 Apr 2017 10:30:16 -0400 Subject: [PATCH 3/8] 0003066: Cache frequent use of AppUtils.getHostName() and AppUtils.getIpAddress() --- .../transport/http/HttpTransportManager.java | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java index f5e29a0704..1a4a261a94 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java @@ -32,6 +32,8 @@ import java.net.URLEncoder; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.zip.GZIPInputStream; import org.apache.commons.codec.binary.Base64; @@ -60,6 +62,10 @@ public class HttpTransportManager extends AbstractTransportManager implements ITransportManager { private ISymmetricEngine engine; + private AtomicReference cachedHostName = new AtomicReference(); + private AtomicReference cachedIpAddress = new AtomicReference(); + private AtomicLong cacheTime = new AtomicLong(-1); + private long hostCacheTtl = 0; public HttpTransportManager() { } @@ -67,6 +73,7 @@ public HttpTransportManager() { public HttpTransportManager(ISymmetricEngine engine) { super(engine.getExtensionService()); this.engine = engine; + hostCacheTtl = engine.getParameterService().getLong("cache.security.token.host.time.ms", 5*60*1000); } public int sendCopyRequest(Node local) throws IOException { @@ -318,10 +325,42 @@ protected String addSecurityToken(String base, String connector, String nodeId, sb.append(WebConstants.SECURITY_TOKEN); sb.append("="); sb.append(securityToken); - append(sb, WebConstants.HOST_NAME, AppUtils.getHostName()); - append(sb, WebConstants.IP_ADDRESS, AppUtils.getIpAddress()); + String[] hostAndIpAddress = getHostAndIpAddress(); + append(sb, WebConstants.HOST_NAME, hostAndIpAddress[0]); + append(sb, WebConstants.IP_ADDRESS, hostAndIpAddress[1]); return sb.toString(); } + + protected String[] getHostAndIpAddress() { + String hostName, ipAddress; + if (cachedHostName.get() == null || cachedIpAddress.get() == null || cacheTimeExpired(cacheTime)) { + cachedHostName.set(null); + cachedIpAddress.set(null); + hostName = AppUtils.getHostName(); + ipAddress = AppUtils.getIpAddress(); + if (!StringUtils.isEmpty(hostName) && !"unknown".equals(hostName)) { + cachedHostName.set(hostName); + } + if (!StringUtils.isEmpty(ipAddress) && !"unknown".equals(ipAddress)) { + cachedIpAddress.set(ipAddress); + } + cacheTime.set(System.currentTimeMillis()); + } else { + hostName = cachedHostName.get(); + ipAddress = cachedIpAddress.get(); + } + + return new String[] {hostName, ipAddress}; + } + + protected boolean cacheTimeExpired(AtomicLong argCacheTime) { + long cacheTimeLong = argCacheTime.get(); + if (cacheTimeLong == -1 || (System.currentTimeMillis()-cacheTimeLong) > hostCacheTtl) { + return true; + } else { + return false; + } + } protected String addNodeId(String base, String nodeId, String connector) { return add(base, WebConstants.NODE_ID, nodeId, connector); From 129e50f58a8a2c145eb741e09cf2f43aafdc72bb Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Mon, 24 Apr 2017 15:46:55 -0400 Subject: [PATCH 4/8] 0003068: MySQL Create Table with Timestamps --- .../db/platform/mysql/MySqlDdlBuilder.java | 3 +++ .../platform/mysql/MySqlDatabasePlatform.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlBuilder.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlBuilder.java index 5dacbf4cdb..63e2483e15 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlBuilder.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlBuilder.java @@ -319,6 +319,9 @@ protected String getSqlType(Column column) { && (column.getMappedTypeCode() == Types.DECIMAL || column.getMappedTypeCode() == Types.NUMERIC)) { sqlType = "BIGINT"; } + if (column.getMappedTypeCode() == Types.TIMESTAMP && column.getScale() > 0) { + sqlType = "TIMESTAMP(" + column.getScale() + ")"; + } return sqlType; } diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDatabasePlatform.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDatabasePlatform.java index 8f71c97825..530f023c35 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDatabasePlatform.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDatabasePlatform.java @@ -1,5 +1,7 @@ package org.jumpmind.db.platform.mysql; +import java.sql.Types; + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -22,6 +24,10 @@ import javax.sql.DataSource; import org.apache.commons.lang.StringUtils; +import org.jumpmind.db.model.Column; +import org.jumpmind.db.model.Database; +import org.jumpmind.db.model.Table; +import org.jumpmind.db.model.TypeMap; import org.jumpmind.db.platform.AbstractJdbcDatabasePlatform; import org.jumpmind.db.platform.DatabaseNamesConstants; import org.jumpmind.db.platform.PermissionResult; @@ -132,4 +138,22 @@ public PermissionResult getCreateSymRoutinePermission() { } return result; } + + @Override + public void makePlatformSpecific(Database database) { + for (Table table : database.getTables()) { + for (Column column : table.getColumns()) { + try { + if (column.getMappedTypeCode() == Types.DATE + && column.findPlatformColumn(DatabaseNamesConstants.ORACLE) != null) { + column.setMappedType(TypeMap.TIMESTAMP); + column.setMappedTypeCode(Types.TIMESTAMP); + column.setScale(6); + } + } + catch (Exception e) {} + } + } + super.makePlatformSpecific(database); + } } From 25849a1fd0685967d086d4f0282f5b31073574a2 Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Tue, 25 Apr 2017 07:45:18 -0400 Subject: [PATCH 5/8] 0003069: Update documentation for 3.8 load balancers to use sticky sessions --- symmetric-assemble/src/asciidoc/installation.ad | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/symmetric-assemble/src/asciidoc/installation.ad b/symmetric-assemble/src/asciidoc/installation.ad index dc8a02bef0..2fcdab35a6 100644 --- a/symmetric-assemble/src/asciidoc/installation.ad +++ b/symmetric-assemble/src/asciidoc/installation.ad @@ -263,8 +263,11 @@ bin/sym_service stop A single SymmetricDS node may be clustered across a series of instances, creating a web farm. A node might be clustered to provide load balancing and failover, for example. -When clustered, a hardware load balancer is typically used -to round robin client requests to the cluster. The load balancer should be configured for stateless connections. +When clustered, a hardware load balancer is typically used. + +For clustered nodes running SymmetricDS 3.8 and later the recommended approach is to configure the load balancer to use sticky sessions or ensure the staging directory for all nodes in the cluster are using a shared network drive. + +For clustered nodes running SymmetricDS 3.7 and earlier it is recommended to round robin client requests to the cluster and configure the load balancer for stateless connections. ifndef::pro[] Also, the `sync.url` (discussed in <>) SymmetricDS property should be set to the URL of the load balancer. From 3abc915c3fa715682bc8a05b1a90b38472f196e9 Mon Sep 17 00:00:00 2001 From: gwilmer Date: Wed, 26 Apr 2017 08:46:43 -0400 Subject: [PATCH 6/8] Fixed #0003070. --- .../java/org/jumpmind/db/platform/AbstractJdbcDdlReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/AbstractJdbcDdlReader.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/AbstractJdbcDdlReader.java index f2c1e337cb..9515f77328 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/AbstractJdbcDdlReader.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/AbstractJdbcDdlReader.java @@ -697,10 +697,10 @@ protected Table readTable(Connection connection, DatabaseMetaDataWrapper metaDat } return table; } catch (RuntimeException ex) { - log.error("Failed to read table: {}", tableName); + log.error(String.format("Failed to read table: %s. Error: %s", tableName, ex.getMessage())); throw ex; } catch (SQLException ex) { - log.error("Failed to read table: {}", tableName); + log.error(String.format("Failed to read table: %s. Error: %s", tableName, ex.getMessage())); throw ex; } } From 0189008e3e0bfe06cd8a7c5542ef948f650be316 Mon Sep 17 00:00:00 2001 From: maxwellpettit Date: Wed, 26 Apr 2017 13:20:08 -0400 Subject: [PATCH 7/8] 0002964: MEDIUMTEXT/LONGTEXT columns being converted to hex representation on mysql data on extract at source node --- .../java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java index d2e09ad199..1915afa595 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java @@ -101,12 +101,14 @@ protected Integer mapUnknownJdbcTypeForColumn(Map values) { String collation = platform.getSqlTemplate().queryForString("select collation_name from information_schema.columns " + "where table_schema = ? and table_name = ? and column_name = ?", catalog, tableName, columnName); - boolean isBinary = collation != null && collation.equalsIgnoreCase("utf8_bin"); + boolean isBinary = collation != null && collation.endsWith("_bin"); if ("LONGTEXT".equals(typeName)) { return isBinary ? Types.BLOB : Types.CLOB; } else if ("MEDIUMTEXT".equals(typeName)) { return isBinary ? Types.BLOB : Types.LONGVARCHAR; + } else if ("TEXT".equals(typeName)) { + return isBinary ? Types.BLOB : Types.LONGVARCHAR; } else if ("TINYTEXT".equals(typeName)) { return isBinary ? Types.BLOB : Types.LONGVARCHAR; } From 36f3b7c5db6cadd9f38dcc390bc2a276e795c654 Mon Sep 17 00:00:00 2001 From: gwilmer Date: Wed, 26 Apr 2017 13:45:57 -0400 Subject: [PATCH 8/8] Fixed #0003076. --- .../jumpmind/db/platform/mysql/MySqlDdlReader.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java index 1915afa595..ec7deb7b40 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/mysql/MySqlDdlReader.java @@ -101,16 +101,20 @@ protected Integer mapUnknownJdbcTypeForColumn(Map values) { String collation = platform.getSqlTemplate().queryForString("select collation_name from information_schema.columns " + "where table_schema = ? and table_name = ? and column_name = ?", catalog, tableName, columnName); - boolean isBinary = collation != null && collation.endsWith("_bin"); + + String convertTextToLobParm = System.getProperty("mysqlddlreader.converttexttolob", + "true"); + boolean convertTextToLob = collation != null && collation.endsWith("_bin") && + convertTextToLobParm.equalsIgnoreCase("true"); if ("LONGTEXT".equals(typeName)) { - return isBinary ? Types.BLOB : Types.CLOB; + return convertTextToLob ? Types.BLOB : Types.CLOB; } else if ("MEDIUMTEXT".equals(typeName)) { - return isBinary ? Types.BLOB : Types.LONGVARCHAR; + return convertTextToLob ? Types.BLOB : Types.LONGVARCHAR; } else if ("TEXT".equals(typeName)) { - return isBinary ? Types.BLOB : Types.LONGVARCHAR; + return convertTextToLob ? Types.BLOB : Types.LONGVARCHAR; } else if ("TINYTEXT".equals(typeName)) { - return isBinary ? Types.BLOB : Types.LONGVARCHAR; + return convertTextToLob ? Types.BLOB : Types.LONGVARCHAR; } return super.mapUnknownJdbcTypeForColumn(values); } else if (type != null && type == Types.OTHER) {