From fa8f5c517ae0c997c55c4fc944459d674b80ac0d Mon Sep 17 00:00:00 2001 From: elong Date: Fri, 1 Sep 2017 10:02:29 -0400 Subject: [PATCH 1/4] add a better log message --- .../java/org/jumpmind/symmetric/service/impl/NodeService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java index ea7c93b691..6c4d775da8 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java @@ -359,8 +359,8 @@ public Node findIdentity(boolean useCache, boolean logSqlError) { cachedNodeIdentity = (Node) getFirstEntry(list); } catch (SqlException ex) { if (logSqlError) { - log.info("Failed to load the node identity because: {}. Returning {}", - ex.getMessage(), cachedNodeIdentity); + log.info("Failed to load the node identity because: {} {}. Returning {}", + ex.getClass().getName(), ex.getMessage(), cachedNodeIdentity); } } } From 0fd5f86f941bfdf5188c4cb0c035529164492a76 Mon Sep 17 00:00:00 2001 From: elong Date: Fri, 1 Sep 2017 11:59:21 -0400 Subject: [PATCH 2/4] 0003210: Turkish Locale Problem --- .../symmetric/db/AbstractTriggerTemplate.java | 9 +++--- .../db/platform/AbstractDatabasePlatform.java | 32 +++++++++---------- .../java/org/jumpmind/util/FormatUtils.java | 29 +++++++++++++++++ .../util/LinkedCaseInsensitiveMap.java | 4 ++- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java index cb9159b5c7..bb0a9e4fca 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java @@ -25,6 +25,7 @@ import java.lang.reflect.Field; import java.sql.Types; +import java.util.Locale; import java.util.Map; import org.apache.commons.lang.NotImplementedException; @@ -374,15 +375,15 @@ public String createTriggerDDL(DataEventType dml, Trigger trigger, TriggerHistor Table table = originalTable.copyAndFilterColumns(history.getParsedColumnNames(), history.getParsedPkColumnNames(), true); - String ddl = sqlTemplates.get(dml.name().toLowerCase() + "TriggerTemplate"); + String ddl = sqlTemplates.get(dml.name().toLowerCase(Locale.US) + "TriggerTemplate"); if (trigger.isStreamRow()) { - String reloadDdl = sqlTemplates.get(dml.name().toLowerCase() + "ReloadTriggerTemplate"); + String reloadDdl = sqlTemplates.get(dml.name().toLowerCase(Locale.US) + "ReloadTriggerTemplate"); if (reloadDdl != null && reloadDdl.length() > 0) { ddl = reloadDdl; } } if (dml.getDmlType().equals(DmlType.UPDATE) && trigger.isUseHandleKeyUpdates()) { - String temp = sqlTemplates.get(dml.name().toLowerCase() + "HandleKeyUpdates" + "TriggerTemplate"); + String temp = sqlTemplates.get(dml.name().toLowerCase(Locale.US) + "HandleKeyUpdates" + "TriggerTemplate"); if (StringUtils.trimToNull(temp)!=null) { ddl=temp; } @@ -402,7 +403,7 @@ public String createPostTriggerDDL(DataEventType dml, Trigger trigger, TriggerHi Table table = originalTable.copyAndFilterColumns(history.getParsedColumnNames(), history.getParsedPkColumnNames(), true); - String ddl = sqlTemplates.get(dml.name().toLowerCase() + "PostTriggerTemplate"); + String ddl = sqlTemplates.get(dml.name().toLowerCase(Locale.US) + "PostTriggerTemplate"); return replaceTemplateVariables(dml, trigger, history, channel, tablePrefix, originalTable, table, defaultCatalog, defaultSchema, ddl); } diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java index 0bdda3f3b5..8f8361840e 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java @@ -419,9 +419,9 @@ protected Object getObjectValue(String value, Column column, BinaryEncoding enco objectValue = parseBigDecimal(value); } else if (type == Types.BOOLEAN) { objectValue = value.equals("1") ? Boolean.TRUE : Boolean.FALSE; - } else if (!(column.getJdbcTypeName() != null && column.getJdbcTypeName().toUpperCase() + } else if (!(column.getJdbcTypeName() != null && FormatUtils.upper(column.getJdbcTypeName()) .contains(TypeMap.GEOMETRY)) - && !(column.getJdbcTypeName() != null && column.getJdbcTypeName().toUpperCase() + && !(column.getJdbcTypeName() != null && FormatUtils.upper(column.getJdbcTypeName()) .contains(TypeMap.GEOGRAPHY)) && (type == Types.BLOB || type == Types.LONGVARBINARY || type == Types.BINARY || type == Types.VARBINARY || @@ -753,7 +753,7 @@ public void prefixDatabase(String prefix, Database targetTables) { boolean storesUpperCaseIdentifiers = isStoresUpperCaseIdentifiers(); for (Table table : tables) { String name = String.format("%s%s", prefix, table.getName()); - table.setName(storesUpperCaseIdentifiers ? name.toUpperCase() : name.toLowerCase()); + table.setName(storesUpperCaseIdentifiers ? FormatUtils.upper(name) : FormatUtils.lower(name)); prefixForeignKeys(table, prefix, storesUpperCaseIdentifiers); prefixIndexes(table, prefix, storesUpperCaseIdentifiers); prefixColumnNames(table, storesUpperCaseIdentifiers); @@ -767,8 +767,8 @@ public void prefixDatabase(String prefix, Database targetTables) { protected void prefixColumnNames(Table table, boolean storesUpperCaseIdentifiers) { Column[] columns = table.getColumns(); for (Column column : columns) { - column.setName(storesUpperCaseIdentifiers ? column.getName().toUpperCase() : column - .getName().toLowerCase()); + column.setName(storesUpperCaseIdentifiers ? FormatUtils.upper(column.getName()) : + FormatUtils.lower(column.getName())); } } @@ -777,22 +777,20 @@ protected void prefixForeignKeys(Table table, String tablePrefix, ForeignKey[] keys = table.getForeignKeys(); for (ForeignKey key : keys) { String prefixedName = tablePrefix + key.getForeignTableName(); - prefixedName = storesUpperCaseIdentifiers ? prefixedName.toUpperCase() : prefixedName - .toLowerCase(); + prefixedName = storesUpperCaseIdentifiers ? FormatUtils.upper(prefixedName) : + FormatUtils.lower(prefixedName); key.setForeignTableName(prefixedName); String keyName = tablePrefix + key.getName(); - keyName = storesUpperCaseIdentifiers ? keyName.toUpperCase() : keyName.toLowerCase(); + keyName = storesUpperCaseIdentifiers ? FormatUtils.upper(keyName) : FormatUtils.lower(keyName); key.setName(keyName); Reference[] refs = key.getReferences(); for (Reference reference : refs) { - reference.setForeignColumnName(storesUpperCaseIdentifiers ? reference - .getForeignColumnName().toUpperCase() : reference.getForeignColumnName() - .toLowerCase()); - reference.setLocalColumnName(storesUpperCaseIdentifiers ? reference - .getLocalColumnName().toUpperCase() : reference.getLocalColumnName() - .toLowerCase()); + reference.setForeignColumnName(storesUpperCaseIdentifiers ? FormatUtils.upper(reference + .getForeignColumnName()) : FormatUtils.lower(reference.getForeignColumnName())); + reference.setLocalColumnName(storesUpperCaseIdentifiers ? FormatUtils.upper(reference + .getLocalColumnName()) : FormatUtils.lower(reference.getLocalColumnName())); } } } @@ -803,8 +801,8 @@ protected void prefixIndexes(Table table, String tablePrefix, boolean storesUppe if (indexes != null) { for (IIndex index : indexes) { String prefixedName = tablePrefix + index.getName(); - prefixedName = storesUpperCaseIdentifiers ? prefixedName.toUpperCase() - : prefixedName.toLowerCase(); + prefixedName = storesUpperCaseIdentifiers ? FormatUtils.upper(prefixedName) + : FormatUtils.lower(prefixedName); index.setName(prefixedName); } } @@ -830,7 +828,7 @@ public String alterCaseToMatchDatabaseDefaultCase(String value) { if (StringUtils.isNotBlank(value)) { boolean storesUpperCase = isStoresUpperCaseIdentifiers(); if (!FormatUtils.isMixedCase(value)) { - value = storesUpperCase ? value.toUpperCase() : value.toLowerCase(); + value = storesUpperCase ? FormatUtils.upper(value) : FormatUtils.lower(value); } } return value; diff --git a/symmetric-util/src/main/java/org/jumpmind/util/FormatUtils.java b/symmetric-util/src/main/java/org/jumpmind/util/FormatUtils.java index 7419e4eeb6..5a0dce6c46 100644 --- a/symmetric-util/src/main/java/org/jumpmind/util/FormatUtils.java +++ b/symmetric-util/src/main/java/org/jumpmind/util/FormatUtils.java @@ -27,6 +27,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TimeZone; import java.util.regex.Matcher; @@ -57,6 +58,12 @@ public final class FormatUtils { public final static int MAX_CHARS_TO_LOG = 1000; private static Pattern pattern = Pattern.compile("\\$\\((.+?)\\)"); + + private static boolean isInfamousTurkey = false; + + static { + isInfamousTurkey = Locale.getDefault().getCountry().equalsIgnoreCase("tr"); + } private FormatUtils() { } @@ -313,4 +320,26 @@ public static String replaceCharsToShortenName(String name) { return name.replaceAll("[^a-zA-Z0-9_]|[a|e|i|o|u|A|E|I|O|U]", ""); } + public static String lower(String str) { + if (isInfamousTurkey) { + return str.toLowerCase(Locale.US); + } + return str.toLowerCase(); + } + + public static String upper(String str) { + if (isInfamousTurkey) { + return str.toUpperCase(Locale.US); + } + return str.toUpperCase(); + } + + public static boolean isInfamousTurkey() { + return isInfamousTurkey; + } + + public static String stripTurkeyDottedI(String str) { + return str.replace("\u0130", "I").replace("\u0131", "i"); + } + } diff --git a/symmetric-util/src/main/java/org/jumpmind/util/LinkedCaseInsensitiveMap.java b/symmetric-util/src/main/java/org/jumpmind/util/LinkedCaseInsensitiveMap.java index ef81d20a13..f259e8ed87 100644 --- a/symmetric-util/src/main/java/org/jumpmind/util/LinkedCaseInsensitiveMap.java +++ b/symmetric-util/src/main/java/org/jumpmind/util/LinkedCaseInsensitiveMap.java @@ -46,7 +46,6 @@ public class LinkedCaseInsensitiveMap extends LinkedHashMap { private final Locale locale; - public LinkedCaseInsensitiveMap(Map values) { this(); putAll(values); @@ -101,6 +100,9 @@ public LinkedCaseInsensitiveMap(int initialCapacity, Locale locale) { @Override public V put(String key, V value) { this.caseInsensitiveKeys.put(convertKey(key), key); + if (FormatUtils.isInfamousTurkey()) { + this.caseInsensitiveKeys.put(FormatUtils.stripTurkeyDottedI(convertKey(key)), key); + } return super.put(key, value); } From ddd4b629b71c234291d7c741b1a0783dfafff724 Mon Sep 17 00:00:00 2001 From: maxwellpettit Date: Tue, 5 Sep 2017 10:59:11 -0400 Subject: [PATCH 3/4] 0003240: Each time a batch is loaded it uses the same thread name. This can be confusing if the same node is being loaded by two thread --- .../symmetric/service/impl/DataLoaderService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java index 63d383616f..988d2cc464 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java @@ -167,6 +167,8 @@ public class DataLoaderService extends AbstractService implements IDataLoaderSer private Date lastUpdateTime; + private CustomizableThreadFactory threadFactory = new CustomizableThreadFactory("dataloader"); + public DataLoaderService(ISymmetricEngine engine) { super(engine.getParameterService(), engine.getSymmetricDialect()); this.incomingBatchService = engine.getIncomingBatchService(); @@ -548,7 +550,8 @@ protected List loadDataFromTransport(final ProcessInfo processInf String targetNodeId = nodeService.findIdentityNodeId(); if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) { processInfo.setStatus(ProcessInfo.Status.TRANSFERRING); - ExecutorService executor = Executors.newFixedThreadPool(1, new CustomizableThreadFactory(String.format("dataloader-%s-%s", sourceNode.getNodeGroupId(), sourceNode.getNodeId()))); + //ExecutorService executor = Executors.newFixedThreadPool(1, new UniqueThreadFactory("dataloader")); + ExecutorService executor = Executors.newFixedThreadPool(1, threadFactory); LoadIntoDatabaseOnArrivalListener loadListener = new LoadIntoDatabaseOnArrivalListener(processInfo, sourceNode.getNodeId(), listener, executor); new SimpleStagingDataWriter(transport.openReader(), stagingManager, Constants.STAGING_CATEGORY_INCOMING, @@ -566,8 +569,9 @@ protected List loadDataFromTransport(final ProcessInfo processInf outWriter.flush(); } } else { - executor.awaitTermination(12, TimeUnit.HOURS); + executor.awaitTermination(12, TimeUnit.HOURS); } + loadListener.isDone(); } else { DataProcessor processor = new DataProcessor(new ProtocolDataReader(BatchType.LOAD, From e0a35899ffa1ae065f409d02d514b235c0c272d5 Mon Sep 17 00:00:00 2001 From: maxwellpettit Date: Tue, 5 Sep 2017 10:59:52 -0400 Subject: [PATCH 4/4] 0003239: Observed that two load threads are loading the same batch at the same time during a pull --- .../jumpmind/symmetric/service/impl/DataLoaderService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java index 988d2cc464..ec27e83100 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java @@ -569,7 +569,11 @@ protected List loadDataFromTransport(final ProcessInfo processInf outWriter.flush(); } } else { - executor.awaitTermination(12, TimeUnit.HOURS); + long hours = 1; + while (!executor.awaitTermination(1, TimeUnit.HOURS)) { + log.info(String.format("Executor has been awaiting loader termination for %d hour(s).", hours)); + hours++; + } } loadListener.isDone();