Skip to content
Permalink
Browse files

Merge branch '3.10' of https://github.com/JumpMind/symmetric-ds.git i…

…nto 3.10
  • Loading branch information...
jumpmind-josh committed May 14, 2019
2 parents 28fd261 + 9646deb commit 173dffd796478b64885419c1fb67336aefb4a978
@@ -1,5 +1,5 @@
symAssembleDir=../symmetric-assemble
version=3.10.0
version=3.10.x-SNAPSHOT
scmVersion=?
deployUser=?
deployPassword=?
@@ -365,7 +365,7 @@ are provided to the script:

|===

.Tranform Expression Example
.Transform Expression Example
====
----
#{currentValue} - #{oldValue} * #{PRICE}
@@ -74,25 +74,17 @@ IMPORTANT: When providing an
endif::pro[]

===== Initial Load Extract In Background

By default, initial loads for a table are broken into multiple batches. SymmetricDS will pre-extract
initial load batches versus having them extracted when
the batch is pulled or pushed. There are two ways to tell
SymmetricDS the number of batches to create for a given table. The first is to specify
a positive integer in the initial_load_batch_count column on
<<Table Routing>>. This
number will dictate the number of batches created for the initial load of the given table.
The second way is to specify 0 for initial_load_batch_count on
<<Table Routing>> and specify a max_batch_size on the reload channel for <<Channels>>.
When 0 is specified for initial_load_batch_count, SymmetricDS will execute a count(*) query on the table during
the extract process and pre-create N batches based on the total number of records found
in the table divided by the `max_batch_size` on the reload channel.

By setting the `initial.load.use.extract.job.enabled` to false all data for a given table will be initial loaded
in a single batch, regardless of the max batch size parameter on the reload channel. That is, for a table with one million
rows, all rows for that table will be initial loaded and sent to the destination node in a
single batch. For large tables, this can result in a batch that can take a long time to
extract and load.

By default, initial loads for a table are broken into multiple batches, with the size of batches based on the
`max_batch_size` of the <<Channels>> for the reload channel being used.
Batches are pre-extracted to staging in the background, instead of waiting for a push or pull to extract them.
An estimated count of rows for the table are queried from the database statistics, or it will execute a count(*) query
from the table if statistics are not available.
The extract process creates batches based on the number of rows in the table divided by the `max_batch_size`.

If the background job is disabled by setting `initial.load.use.extract.job.enabled` to false,
then all data for a given table will be extracted into a single batch during a push or pull, regardless of channel settings.
For large tables, this can result in a batch that can take a long time to extract and load.

ifndef::pro[]
===== Reverse Initial Loads
@@ -32,7 +32,7 @@ endif::pro[]
ifdef::pro[]
Download and install SymmetricDS from the http://www.jumpmind.com/products/symmetricds/download[JumpMind] website.

TIP: For complete installation instructions click link:../user-guide.html#_installation[here,window="_tutorials"]
TIP: For complete installation instructions click link:user-guide.html#_installation[here,window="_tutorials"]
endif::pro[]


@@ -181,7 +181,7 @@ image::wizard-node-setup.png[]
. Provide a password for the web console, hit next
. Finish

TIP: For complete instructions creating the master node click link:../user-guide.html#_master_node_setup[here,window="_tutorials"]
TIP: For complete instructions creating the master node click link:user-guide.html#_master_node_setup[here,window="_tutorials"]

endif::pro[]

@@ -268,7 +268,7 @@ image::wizard-config/wizard-config-client-server.png[]
. Select tables you wish to sync from server to client, hit next
. Verify the summary, hit finish

TIP: For complete instructions for the quick config wizard click link:../user-guide.html#_quick_config_wizard[here,window="_tutorials"]
TIP: For complete instructions for the quick config wizard click link:user-guide.html#_quick_config_wizard[here,window="_tutorials"]
endif::pro[]


@@ -305,7 +305,7 @@ image::wizard-connect/wizard-connect-menu.png[]
. Choose appropriate options here depending if your target tables already exist and if they should be initially loaded with data from the source. Hit next
. Hit finish

TIP: For complete instructions to setup the target database click link:../user-guide.html#_add_node[here,window="_tutorials"]
TIP: For complete instructions to setup the target database click link:user-guide.html#_add_node[here,window="_tutorials"]

endif::pro[]

@@ -406,7 +406,10 @@ public static File createSnapshot(ISymmetricEngine engine) {

File jarFile = null;
try {
jarFile = new File(getSnapshotDirectory(engine), tmpDir.getName() + ".zip");
String filename = tmpDir.getName() + ".zip";
if (parameterService.is(ParameterConstants.SNAPSHOT_FILE_INCLUDE_HOSTNAME))
filename = AppUtils.getHostName() + "_" + filename;
jarFile = new File(getSnapshotDirectory(engine), filename);
ZipBuilder builder = new ZipBuilder(tmpDir, jarFile, new File[] { tmpDir });
builder.build();
FileUtils.deleteDirectory(tmpDir);
@@ -487,6 +487,8 @@ private ParameterConstants() {
public final static String CLOUD_BULK_LOAD_AZURE_BLOB_CONTAINER = "cloud.bulk.load.azure.blob.container";
public final static String CLOUD_BULK_LOAD_AZURE_SAS_TOKEN = "cloud.bulk.load.azure.sas.token";

public final static String SNAPSHOT_FILE_INCLUDE_HOSTNAME = "snapshot.file.include.hostname";

public static Map<String, ParameterMetaData> getParameterMetaData() {
return parameterMetaData;
}
@@ -562,13 +562,17 @@ initial.load.transport.max.bytes.to.sync=524288000
initial.load.unblock.channels.on.error=false

# Set this if tables should be created prior to an initial load.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
# Type: boolean
initial.load.create.first=false

# Set this if tables should be purged prior to an initial load.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
@@ -577,6 +581,8 @@ initial.load.delete.first=false

# This is the SQL statement that will be used for purging a table
# during an initial load.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
@@ -585,6 +591,8 @@ initial.load.delete.first.sql=delete from %s
# Indicate that the initial load events should be put on the reload channel.
# If this is set to false each table will be put on it's assigned channel during
# the reload.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
@@ -657,7 +665,9 @@ initial.load.use.column.templates.enabled=false
# a single line that starts with delimiter and is followed by the new delimiter, then the old
# For example a line that reads:
# delimiter $;
# would change sql lines to have a delimiter of $ for subsequent lines
# would change sql lines to have a delimiter of $ for subsequent lines.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
@@ -669,7 +679,10 @@ initial.load.before.sql=
# a single line that starts with delimiter and is followed by the new delimiter, then the old
# For example a line that reads:
# delimiter $;
# would change sql lines to have a delimiter of $ for subsequent lines
# would change sql lines to have a delimiter of $ for subsequent lines.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
# Type: sql
@@ -727,6 +740,8 @@ initial.load.purge.stage.immediate.threshold.rows=5000
# If tables are created as part of the initial load, it will defer the creation
# of foreign keys and indexes to improve performance.
# After data is loaded, the constraints will be added after tables have reached consistency.
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
@@ -750,6 +765,8 @@ initial.load.recursion.self.fk=true
auto.registration=false

# If this is true, a reload is automatically sent to nodes when they register
# This parameter needs set for the node that will send the initial load,
# not the node receiving it.
#
# DatabaseOverridable: true
# Tags: load
@@ -2505,3 +2522,10 @@ cloud.bulk.load.azure.blob.container=symmetricds
# DatabaseOverridable: true
# Tags: cloud, snowflake, azure
cloud.bulk.load.azure.sas.token=

# Prefix snapshot filename with hostname to indicate which environment it came from
#
# DatabaseOverridable: true
# Tags: other
# Type: boolean
snapshot.file.include.hostname=false
@@ -245,7 +245,7 @@ public void write(CsvData data) {
if (log.isDebugEnabled()) {
log.debug(
"Transform indicated that the target row should be ignored with a target key of: {}",
"unknown. Transformation aborted during tranformation of key");
"unknown. Transformation aborted during transformation of key");
}
return new ArrayList<TransformedData>(0);
} finally {
@@ -26,11 +26,14 @@

import javax.sql.DataSource;

import org.jumpmind.db.model.TypeMap;
import org.jumpmind.db.platform.DatabaseInfo;
import org.jumpmind.db.sql.JdbcSqlTemplate;
import org.jumpmind.db.sql.SqlTemplateSettings;
import org.jumpmind.db.sql.SymmetricLobHandler;
import org.springframework.jdbc.core.SqlTypeValue;
import org.springframework.jdbc.core.StatementCreatorUtils;
import org.springframework.jdbc.support.lob.LobHandler;

public class PostgreSqlJdbcSqlTemplate extends JdbcSqlTemplate {

@@ -78,5 +81,23 @@ public boolean isDataTruncationViolation(Throwable ex) {
}
return dataTruncationViolation;
}

@Override
protected void setBitValue(PreparedStatement ps, int i, Object arg, int argType) throws SQLException {
if(argType == Types.BIT && arg != null && arg instanceof Number) {
Number n = (Number) arg;
if(n.intValue() > 0) {
StatementCreatorUtils.setParameterValue(ps, i, Types.VARCHAR, "1");
} else if(n.intValue() == 0) {
StatementCreatorUtils.setParameterValue(ps, i, Types.VARCHAR, "0");
} else {
StatementCreatorUtils.setParameterValue(ps, i, verifyArgType(arg, argType), arg);
}
} else {
StatementCreatorUtils.setParameterValue(ps, i, verifyArgType(arg, argType), arg);
}
}



}
@@ -1075,7 +1075,9 @@ public void setValues(PreparedStatement ps, Object[] args, int[] argTypes,
} else if ((argType == Types.DECIMAL || argType == Types.NUMERIC) && arg != null) {
setDecimalValue(ps, i, arg, argType);
} else if (argType == Types.TINYINT) {
setTinyIntValue(ps, i, arg, argType);
setTinyIntValue(ps, i, arg, argType);
} else if (argType == Types.BIT && arg instanceof Number) {
setBitValue(ps, i, arg, argType);
} else {
StatementCreatorUtils.setParameterValue(ps, i, verifyArgType(arg, argType), arg);
}
@@ -1098,6 +1100,10 @@ protected void setDecimalValue(PreparedStatement ps, int i, Object arg, int argT
}
}

protected void setBitValue(PreparedStatement ps, int i, Object arg, int argType) throws SQLException {
StatementCreatorUtils.setParameterValue(ps, i, verifyArgType(arg, argType), arg);
}

protected void setNanOrNull(PreparedStatement ps, int i, Object arg, int argType) throws SQLException {
StatementCreatorUtils.setParameterValue(ps, i, verifyArgType(arg, argType), null);
}
@@ -354,7 +354,7 @@ public void install() {
service = advapi.CreateService(manager, config.getName(), config.getDisplayName(), Winsvc.SERVICE_ALL_ACCESS,
WinsvcEx.SERVICE_WIN32_OWN_PROCESS, config.isAutoStart() || config.isDelayStart() ? WinsvcEx.SERVICE_AUTO_START
: WinsvcEx.SERVICE_DEMAND_START, WinsvcEx.SERVICE_ERROR_NORMAL,
commandToString(getWrapperCommand("init")), null, null, dependencies, null, null);
commandToString(getWrapperCommand("init", true)), null, null, dependencies, null, null);

if (service != null) {
Advapi32Ex.SERVICE_DESCRIPTION desc = new Advapi32Ex.SERVICE_DESCRIPTION(config.getDescription());
@@ -35,7 +35,8 @@ public static void main(String[] args) throws Exception {

String appDir = null;
String configFile = null;
String jarFile = Wrapper.class.getProtectionDomain().getCodeSource().getLocation().getFile();
// Decode spaces and other special characters
String jarFile = Wrapper.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath();

if (args.length == 2) {
configFile = args[1];
@@ -73,7 +73,7 @@ public void start() {

stopProcesses(true);
System.out.println("Waiting for server to start");
ArrayList<String> cmdLine = getWrapperCommand("exec");
ArrayList<String> cmdLine = getWrapperCommand("exec", false);
Process process = null;
boolean success = false;
int rc = 0;
@@ -346,15 +346,18 @@ protected String commandToString(ArrayList<String> cmd) {
return sb.toString();
}

protected ArrayList<String> getWrapperCommand(String arg) {
protected ArrayList<String> getWrapperCommand(String arg, boolean isQuotedArguments) {
ArrayList<String> cmd = new ArrayList<String>();
String quote = getWrapperCommandQuote();

String quote = isQuotedArguments ? getWrapperCommandQuote() : "";

cmd.add(quote + config.getJavaCommand() + quote);
cmd.add("-Djava.io.tmpdir="+quote+System.getProperty("java.io.tmpdir")+quote);
cmd.add("-jar");
cmd.add(quote + config.getWrapperJarPath() + quote);
cmd.add(arg);
cmd.add(quote + config.getConfigFile() + quote);

return cmd;
}

0 comments on commit 173dffd

Please sign in to comment.
You can’t perform that action at this time.