Skip to content

Commit

Permalink
Merge branch '3.8' of https://github.com/JumpMind/symmetric-ds.git in…
Browse files Browse the repository at this point in the history 
…to 3.8
  • Loading branch information
gregwilmer committed Dec 1, 2017
2 parents 70f837d + 5f3dab1 commit c0d2f71
Show file tree
Hide file tree
Showing 18 changed files with 506 additions and 262 deletions.
31 changes: 20 additions & 11 deletions symmetric-assemble/asciidoc.gradle
View file
Expand Up @@ -61,16 +61,30 @@ task generateAppendixes {
}
}

ext.getDocDir = {
if (project.properties.containsKey('pro')) {
return file("$buildDir/../../../symmetric-pro/symmetric-pro/src/main/resources/VAADIN/doc")
} else {
return file("$buildDir/doc")
}
}

task cleanDocs {
dependsOn asciidoctor
def destinationDir = getDocDir()
doFirst {
println "Deleting $destinationDir.path"
delete destinationDir
}
}

task generateDocs(type: Copy) {

dependsOn asciidoctor
dependsOn cleanDocs
group = 'SymmetricDS'
description = 'Generates user manual and documentation'

destinationDir = file("$buildDir/doc")
if (project.properties.containsKey('pro')) {
destinationDir = file("$buildDir/../../../symmetric-pro/symmetric-pro/src/main/resources/VAADIN/doc")
}
destinationDir = getDocDir()

into('html') {
from ("$buildDir/src/asciidoc/html5") { include "user-guide.html" }
Expand Down Expand Up @@ -113,12 +127,7 @@ task generateDocs(type: Copy) {

into('pdf') {
from ("$buildDir/src/asciidoc/pdf/operations") { include "operations.pdf" }
}

doFirst {
println "Deleting $destinationDir.path"
delete destinationDir
}
}

}

Expand Down
2 changes: 1 addition & 1 deletion symmetric-client/src/main/java/org/jumpmind/symmetric/db/ase/AseTriggerTemplate.java
View file
Expand Up @@ -47,7 +47,7 @@ public AseTriggerTemplate(ISymmetricDialect symmetricDialect) {
stringColumnTemplate = "case when $(tableAlias)." + quote + "$(columnName)" + quote + " is null then null else '\"' + str_replace(str_replace($(tableAlias)." + quote + "$(columnName)" + quote + ",'\\','\\\\'),'\"','\\\"') + '\"' end" ;
numberColumnTemplate = "case when $(tableAlias)." + quote + "$(columnName)" + quote + " is null then null else ('\"' + convert(varchar,$(tableAlias)." + quote + "$(columnName)" + quote + ") + '\"') end" ;
datetimeColumnTemplate = "case when $(tableAlias)." + quote + "$(columnName)" + quote + " is null then null else ('\"' + str_replace(convert(varchar,$(tableAlias)." + quote + "$(columnName)" + quote + ",102),'.','-') + ' ' + right('00'+convert(varchar,datepart(HOUR,$(tableAlias)." + quote + "$(columnName)" + quote + ")),2)+':'+right('00'+convert(varchar,datepart(MINUTE,$(tableAlias)." + quote + "$(columnName)" + quote + ")),2)+':'+right('00'+convert(varchar,datepart(SECOND,$(tableAlias)." + quote + "$(columnName)" + quote + ")),2)+'.'+right('000'+convert(varchar,datepart(MILLISECOND,$(tableAlias)." + quote + "$(columnName)" + quote + ")),3) + '\"') end" ;
clobColumnTemplate = "case when datalength($(origTableAlias)." + quote + "$(columnName)" + quote + ") is null or datalength($(origTableAlias)." + quote + "$(columnName)" + quote + ")=0 then null else '\"' + str_replace(str_replace(cast($(origTableAlias)." + quote + "$(columnName)" + quote + " as varchar(16384)),'\\','\\\\'),'\"','\\\"') + '\"' end" ;
clobColumnTemplate = "case when datalength($(origTableAlias)." + quote + "$(columnName)" + quote + ") is null or datalength($(origTableAlias)." + quote + "$(columnName)" + quote + ")=0 then null when char_length($(origTableAlias)." + quote + "$(columnName)" + quote + ") > 16384 then '\"\\b\"' else '\"' + str_replace(str_replace(cast($(origTableAlias)." + quote + "$(columnName)" + quote + " as varchar(16384)),'\\','\\\\'),'\"','\\\"') + '\"' end" ;
blobColumnTemplate = "case when $(origTableAlias)." + quote + "$(columnName)" + quote + " is null then null else '\"' + bintostr(convert(varbinary(16384),$(origTableAlias)." + quote + "$(columnName)" + quote + ")) + '\"' end" ;
binaryColumnTemplate = "case when $(tableAlias)." + quote + "$(columnName)" + quote + " is null then null else '\"' + bintostr(convert(varbinary(16384),$(tableAlias)." + quote + "$(columnName)" + quote + ")) + '\"' end" ;
imageColumnTemplate = "case when datalength($(origTableAlias)." + quote + "$(columnName)" + quote + ") is null or datalength($(origTableAlias)." + quote + "$(columnName)" + quote + ")=0 then null else '\"' + bintostr(convert(varbinary(16384),$(origTableAlias)." + quote + "$(columnName)" + quote + ")) + '\"' end" ;
Expand Down
3 changes: 2 additions & 1 deletion ...tric-client/src/main/java/org/jumpmind/symmetric/service/impl/ClientExtensionService.java
View file
Expand Up @@ -41,7 +41,8 @@ public ClientExtensionService(ISymmetricEngine engine, ApplicationContext spring
this.springContext = springContext;
}

public void refresh() {
@Override
public synchronized void refresh() {
super.refresh();

if (springContext != null) {
Expand Down
243 changes: 110 additions & 133 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/util/SnapshotUtil.java
View file

Large diffs are not rendered by default.

108 changes: 66 additions & 42 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java
View file
Expand Up @@ -21,6 +21,7 @@
package org.jumpmind.symmetric.route;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
Expand Down Expand Up @@ -97,6 +98,8 @@ public class DataGapFastDetector extends DataGapDetector implements ISqlRowMappe
protected long earliestTransactionTime = 0;

protected boolean supportsTransactionViews;

protected static Map<String, Boolean> firstTime = Collections.synchronizedMap(new HashMap<String, Boolean>());

public DataGapFastDetector(IDataService dataService, IParameterService parameterService, IContextService contextService,
ISymmetricDialect symmetricDialect, IRouterService routerService, IStatisticManager statisticManager, INodeService nodeService) {
Expand All @@ -110,35 +113,39 @@ public DataGapFastDetector(IDataService dataService, IParameterService parameter
}

public void beforeRouting() {
maxDataToSelect = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE);
detectInvalidGaps = parameterService.is(ParameterConstants.ROUTING_DETECT_INVALID_GAPS);
reset();

if (isFullGapAnalysis()) {
ProcessInfo processInfo = this.statisticManager.newProcessInfo(new ProcessInfoKey(
nodeService.findIdentityNodeId(), null, ProcessType.GAP_DETECT));
processInfo.setStatus(Status.QUERYING);
log.info("Full gap analysis is running");
long ts = System.currentTimeMillis();
gaps = dataService.findDataGaps();
if (detectInvalidGaps) {
fixOverlappingGaps(gaps, processInfo);
}
queryDataIdMap();
processInfo.setStatus(Status.OK);
log.info("Querying data in gaps from database took {} ms", System.currentTimeMillis() - ts);
afterRouting();
try {
maxDataToSelect = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE);
detectInvalidGaps = parameterService.is(ParameterConstants.ROUTING_DETECT_INVALID_GAPS) || firstTime.get(parameterService.getEngineName()) == null;
reset();
log.info("Full gap analysis is done after {} ms", System.currentTimeMillis() - ts);
} else if (gaps == null || parameterService.is(ParameterConstants.CLUSTER_LOCKING_ENABLED)) {
ProcessInfo processInfo = this.statisticManager.newProcessInfo(new ProcessInfoKey(
nodeService.findIdentityNodeId(), null, ProcessType.GAP_DETECT));
processInfo.setStatus(Status.QUERYING);
gaps = dataService.findDataGaps();
if (detectInvalidGaps) {
fixOverlappingGaps(gaps, processInfo);

if (isFullGapAnalysis()) {
ProcessInfo processInfo = this.statisticManager
.newProcessInfo(new ProcessInfoKey(nodeService.findIdentityNodeId(), null, ProcessType.GAP_DETECT));
processInfo.setStatus(Status.QUERYING);
log.info("Full gap analysis is running");
long ts = System.currentTimeMillis();
gaps = dataService.findDataGaps();
if (detectInvalidGaps) {
fixOverlappingGaps(gaps, processInfo);
}
queryDataIdMap();
processInfo.setStatus(Status.OK);
log.info("Querying data in gaps from database took {} ms", System.currentTimeMillis() - ts);
afterRouting();
reset();
log.info("Full gap analysis is done after {} ms", System.currentTimeMillis() - ts);
} else if (gaps == null || parameterService.is(ParameterConstants.CLUSTER_LOCKING_ENABLED)) {
ProcessInfo processInfo = this.statisticManager
.newProcessInfo(new ProcessInfoKey(nodeService.findIdentityNodeId(), null, ProcessType.GAP_DETECT));
processInfo.setStatus(Status.QUERYING);
gaps = dataService.findDataGaps();
if (detectInvalidGaps) {
fixOverlappingGaps(gaps, processInfo);
}
processInfo.setStatus(Status.OK);
}
processInfo.setStatus(Status.OK);
} finally {
firstTime.put(parameterService.getEngineName(), true);
}
}

Expand Down Expand Up @@ -321,13 +328,19 @@ protected boolean addDataGap(DataGap dataGap) {
gapsAdded.add(dataGap);
gapsAll.add(dataGap);
} else {
log.info("Data IDs: " + dataIds.toString());
log.info("Data Gaps: " + gaps.toString());
log.info("Added Data Gaps: " + gapsAdded.toString());
log.info("Deleted Data Gaps: " + gapsDeleted.toString());
printGapState();
}
return isOkay;
}

private void printGapState() {
StringBuilder buff = new StringBuilder();
buff.append("\nData IDs: " + dataIds).append("\n");
buff.append("Data Gaps: " + gaps).append("\n");
buff.append("Added Data Gaps: " + gapsAdded) .append("\n");
buff.append("Deleted Data Gaps: " + gapsDeleted).append("\n");
log.info(buff.toString());
}

protected long saveDataGaps(long ts, long printStats) {
ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
Expand All @@ -345,7 +358,7 @@ protected long saveDataGaps(long ts, long printStats) {
log.info("There are {} data gap changes, which is within the max of {}, so switching to database",
totalGapChanges, maxGapChanges);
useInMemoryGaps = false;
printStats = insertDataGaps(transaction, ts, printStats);
printStats = insertDataGaps(transaction, ts, printStats, gaps);
} else {
if (!useInMemoryGaps) {
log.info("There are {} data gap changes, which exceeds the max of {}, so switching to in-memory",
Expand All @@ -357,7 +370,7 @@ protected long saveDataGaps(long ts, long printStats) {
}
} else {
printStats = deleteDataGaps(transaction, ts, printStats);
printStats = insertDataGaps(transaction, ts, printStats);
printStats = insertDataGaps(transaction, ts, printStats, gapsAdded);
}
transaction.commit();
} catch (Error ex) {
Expand Down Expand Up @@ -392,10 +405,10 @@ protected long deleteDataGaps(ISqlTransaction transaction, long ts, long printSt
}
return printStats;
}

protected long insertDataGaps(ISqlTransaction transaction, long ts, long printStats) {
protected long insertDataGaps(ISqlTransaction transaction, long ts, long printStats, Collection<DataGap> argGaps) {
int counter = 0;
for (DataGap dataGap : gapsAdded) {
for (DataGap dataGap : argGaps) {
dataService.insertDataGap(transaction, dataGap);
counter++;
if (System.currentTimeMillis() - printStats > 30000) {
Expand Down Expand Up @@ -451,27 +464,31 @@ protected Map<DataGap, List<Long>> getDataIdMap() {
return map;
}

protected void fixOverlappingGaps(List<DataGap> gaps, ProcessInfo processInfo) {
protected void fixOverlappingGaps(List<DataGap> gapsToCheck, ProcessInfo processInfo) {
List<DataGap> gapsCopy = new ArrayList<DataGap>(gapsToCheck);
boolean ok = true;
try {
ISqlTransaction transaction = null;
log.debug("Looking for overlapping gaps");
try {
ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
transaction = sqlTemplate.startSqlTransaction();
DataGap prevGap = null, lastGap = null;
for (int i = 0; i < gaps.size(); i++) {
DataGap curGap = gaps.get(i);
for (int i = 0; i < gapsCopy.size(); i++) {
DataGap curGap = gapsCopy.get(i);
if (lastGap != null) {
ok = false;
log.warn("Removing gap found after last gap: " + curGap);
dataService.deleteDataGap(transaction, curGap);
gaps.remove(i--);
gapsCopy.remove(i--);
} else {
if (lastGap == null && curGap.gapSize() >= maxDataToSelect - 1) {
lastGap = curGap;
}

if (prevGap != null) {
if (prevGap.overlaps(curGap)) {
ok = false;
log.warn("Removing overlapping gaps: " + prevGap + ", " + curGap);
dataService.deleteDataGap(transaction, prevGap);
dataService.deleteDataGap(transaction, curGap);
Expand All @@ -484,15 +501,21 @@ protected void fixOverlappingGaps(List<DataGap> gaps, ProcessInfo processInfo) {
}
log.warn("Inserting new gap to fix overlap: " + newGap);
dataService.insertDataGap(transaction, newGap);
gaps.remove(i--);
gaps.set(i, newGap);
gapsCopy.remove(i--);
gapsCopy.set(i, newGap);
curGap = newGap;
}
}
}
prevGap = curGap;
}
transaction.commit();
if (!ok) {
printGapState();
log.info("Fixed gaps: " + gapsCopy);
}
gaps.clear();
gaps.addAll(gapsCopy);
} catch (Error ex) {
if (transaction != null) {
transaction.rollback();
Expand All @@ -514,6 +537,7 @@ protected void fixOverlappingGaps(List<DataGap> gaps, ProcessInfo processInfo) {
}
}


public Long mapRow(Row row) {
return row.getLong("data_id");
}
Expand Down

0 comments on commit c0d2f71

Please sign in to comment.