Merge branch '3.9' of https://github.com/JumpMind/symmetric-ds.git into

3.9

Conflicts:
	symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java
	symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java

symmetric-android/src/main/java/org/jumpmind/symmetric/android/AndroidDatabasePlatform.java

@@ -22,8 +22,12 @@
 
 import java.math.BigDecimal;
 
+import org.jumpmind.db.model.Database;
 import org.jumpmind.db.platform.AbstractDatabasePlatform;
 import org.jumpmind.db.platform.DatabaseNamesConstants;
+import org.jumpmind.db.platform.PermissionResult;
+import org.jumpmind.db.platform.PermissionType;
+import org.jumpmind.db.platform.PermissionResult.Status;
 import org.jumpmind.db.platform.sqlite.SqliteDdlBuilder;
 import org.jumpmind.db.platform.sqlite.SqliteDdlReader;
 import org.jumpmind.db.sql.ISqlTemplate;
@@ -110,4 +114,27 @@ protected Object parseInteger(String value) {
         }
     }
 
+    @Override
+    public PermissionResult getCreateSymTablePermission(Database database) {     
+        PermissionResult result = new PermissionResult(PermissionType.CREATE_TABLE, Status.UNIMPLEMENTED);
+        return result;
+    }
+
+    @Override
+    public PermissionResult getDropSymTablePermission() {     
+        PermissionResult result = new PermissionResult(PermissionType.DROP_TABLE, Status.UNIMPLEMENTED);
+        return result;
+    }
+
+    @Override
+    public PermissionResult getAlterSymTablePermission(Database database) {     
+        PermissionResult result = new PermissionResult(PermissionType.ALTER_TABLE, Status.UNIMPLEMENTED);
+        return result;
+    }
+
+    @Override
+    public PermissionResult getDropSymTriggerPermission() {     
+        PermissionResult result = new PermissionResult(PermissionType.DROP_TRIGGER, Status.UNIMPLEMENTED);
+        return result;
+    }
 }

symmetric-android/src/main/java/org/jumpmind/symmetric/android/AndroidJobManager.java

@@ -59,10 +59,17 @@ public class AndroidJobManager implements IJobManager {
     protected long lastFileSyncTrackerTime = System.currentTimeMillis();
 
     protected long lastFileSyncPushTime = System.currentTimeMillis();    
+
+    protected boolean started = false;
 
     public AndroidJobManager(ISymmetricEngine engine) {
         this.engine = engine;
     }
+
+    @Override
+    public boolean isStarted() {
+        return started;
+    }
 
     public List<IJob> getJobs() {
         List<IJob> jobs = new ArrayList<IJob>(1);
@@ -77,12 +84,14 @@ public void startJobs() {
             job = new Job();
             job.start();
         }
+        started = true;
     }
 
     public void stopJobs() {
         if (job != null) {
             job.stop();
         }
+        started = false;
     }
 
     public IJob getJob(String name) {

symmetric-android/src/main/java/org/jumpmind/symmetric/android/AndroidSqlTransaction.java

@@ -27,6 +27,7 @@
 import org.jumpmind.db.model.Table;
 import org.jumpmind.db.sql.ISqlRowMapper;
 import org.jumpmind.db.sql.ISqlTransaction;
+import org.jumpmind.db.sql.ISqlTransactionListener;
 import org.jumpmind.db.sql.Row;
 import org.jumpmind.db.sql.mapper.RowMapper;
 
@@ -43,13 +44,20 @@ public class AndroidSqlTransaction implements ISqlTransaction {
     protected String sql;
 
     protected boolean needsRolledback = false;
+
+    protected List<ISqlTransactionListener> listeners = new ArrayList<ISqlTransactionListener>();
 
     public AndroidSqlTransaction(AndroidSqlTemplate sqlTemplate, boolean autoCommit) {
         this.autoCommit = autoCommit;
         this.sqlTemplate = sqlTemplate;
         this.database = sqlTemplate.getDatabaseHelper().getWritableDatabase();
         this.database.beginTransaction();
     }
+
+    @Override
+    public void addSqlTransactionListener(ISqlTransactionListener listener) {
+        listeners.add(listener);
+    }
 
     public void setInBatchMode(boolean batchMode) {
     }
@@ -83,6 +91,10 @@ public <T> T queryForObject(String sql, Class<T> clazz, Object... args) {
     public void commit() {
         try {
             this.database.setTransactionSuccessful();
+
+            for (ISqlTransactionListener listener : listeners) {
+                listener.transactionCommitted();
+            }
         } catch (Exception ex) {
             throw this.sqlTemplate.translate(ex);
         } finally {
@@ -93,6 +105,10 @@ public void commit() {
 
     public void rollback() {
         needsRolledback = true;
+
+        for (ISqlTransactionListener listener : listeners) {
+            listener.transactionRolledBack();
+        }
     }
 
     public void close() {

symmetric-assemble/.gitignore

@@ -7,3 +7,4 @@
 /.classpath
 /build/
 /bin/
+/.DS_Store

symmetric-assemble/build.gradle

@@ -9,10 +9,18 @@ buildscript {
         classpath 'org.asciidoctor:asciidoctorj-pdf:1.5.0-alpha.6'
         classpath 'org.springframework.build.gradle:propdeps-plugin:0.0.7'
         classpath 'io.spring.gradle:docbook-reference-plugin:0.3.0'
-        classpath "com.smokejumperit.gradle.license:Gradle-License-Report:0.0.2"
+        classpath ("com.smokejumperit.gradle.license:Gradle-License-Report:0.0.2") {
+            // guava 21 requires Java 8, setting up for a clean build with only Java 1.7 tools.
+            exclude group: 'com.google.guava', module: 'guava'
+        }
+        classpath 'com.google.guava:guava:20.0'
     }
 }
 
+task buildScriptDependencies(type: org.gradle.api.tasks.diagnostics.DependencyReportTask) {
+    configurations = project.buildscript.configurations
+}
+
 project.buildDir='target'
 apply from: symAssembleDir + '/common.gradle'
 
@@ -267,10 +275,6 @@ project(':symmetric-server') {
                     include 'bin/setenv'
                     fileMode 0755
                 }
-                from('../symmetric-core/src/main/resources/symmetric-default.properties') {
-                    rename 'symmetric-default.properties', 'conf/symmetric.properties'
-                    fileMode 0664
-                }
                 into('doc') { from ("$symAssembleDir/target/doc") }
                 into('lib') {
                     from project(":symmetric-wrapper").configurations.compile {

symmetric-assemble/src/asciidoc/advanced-topics.ad

@@ -237,12 +237,13 @@ communication will be secured (typically the "root" or "central office" server).
 ----
 keytool -keystore keystore -delete -alias sym
 ----
+IMPORTANT: If you receive a message like "keytool error: java.io.IOException: Invalid keystore format," try adding a parameter of "-storetype jceks". Or, if you receive a message like, "keytool error: java.lang.Exception: Alias <sym> does not exist" - then the alias does not exist and you can skip this step.
 
 [source, cli]
 ----
 keytool -keystore cacerts -delete -alias sym
 ----
-
+IMPORTANT: See above for possible errors from this command.
 [source]
 ----
 Enter keystore password:  changeit

symmetric-assemble/src/asciidoc/appendix/firebird.ad

@@ -27,6 +27,7 @@ exceed 64KB.  For change capture, the row_data and old_data are limited to 10KB
 for performance reasons.  If you get the error of "arithmetic exception, numeric overflow, or string truncation" during
 extraction of a batch, set the contains_big_lob to true for the channel.
 
+Firebird 3 is supported, however legacy authentication must be enabled in order to connect.  Please refer to the Firebird 3.0 documentation for instructions on enabling this feature.
 
 .Supported Data Types
 |===

symmetric-assemble/src/asciidoc/appendix/mssqlserver.ad

@@ -15,6 +15,10 @@ If you're unable to change NOCOUNT for the server, the "db.init.sql" parameter c
 Connections are pooled and expected to be in the database context like a new connection, so avoid using the "USE database" 
 Transact-SQL statement in extension code.
 
+If SQL Server is configured with a default collation that does NOT support unicode then we have experienced bad performance for update and delete statements when 
+a table has character based primary keys.  This is because statements are prepared for a unicode type and as a result the indexes are not used.  You can turn this 
+functionality off in JTDS by appending the following to your db.url:  ;sendStringParametersAsUnicode=false
+
 ==== SQL Server Permissions
 
 SymmetricDS can be configured to work with a variety SQL Server configurations.  It is possible to install SymmetricDS in the same database and schema with the same user account your application runs.   

symmetric-assemble/src/asciidoc/appendix/postgresql.ad

@@ -8,13 +8,14 @@ Starting with PostgreSQL 8.3, SymmetricDS supports the transaction identifier.
 Binary Large Object (BLOB) replication is supported for both byte array (BYTEA) and object ID (OID) data types.
 
 In order to function properly, SymmetricDS needs to use session variables.
-On PostgreSQL, session variables are enabled using a custom variable class. Add the following line to the postgresql.conf file of PostgreSQL server:
+Before PostgreSQL 9.2, session variables are enabled using a custom variable class. Add the following line to the postgresql.conf file of PostgreSQL server on versions before 9.2:
 
 ----
 custom_variable_classes = 'symmetric'
 ----
 
-This setting is required, and SymmetricDS will log an error and exit if it cannot set session variables.
+This setting is required on versions before 9.2, and SymmetricDS will log an error and exit if it cannot set session variables. 
+PostgreSQL versions 9.2 or later do not require this setting.
 
 Before database triggers can be created by in PostgreSQL, the plpgsql language handler must be installed on the database. 
 If plpgsql is not already installed, the following statements can be run by the administrator on the database:

symmetric-assemble/src/asciidoc/configuration/channels.ad

@@ -47,18 +47,10 @@ Data loaders can also use other techniques to increase performance of data loads
 
 |ftp_localhost|Sends the data in CSV format to a configured ftp location.  These locations are setup in the TODO {SYM_HOME}/conf/ftp-extensions.xml
 
-|mysql_bulk|MySQL bulk loader.  
-
-|mssql_bulk|Microsoft SQL Server bulk loader.  
-
-|postgres_bulk|PostgreSQL bulk loader.  
-
-|oracle_bulk|Oracle bulk loader. 
+|bulk|Assigns the appropriate bulk loader to this channel.  Supported bulk loaders include:  Microsoft SQL, PostgreSQL, MySQL and Amazon Redshift over S3. 
 
 |mongodb|MongoDB data loader.
 
-|redshift_bulk|Amazon Redshift bulk loader over S3.
-
 |===
 
 TIP: Tables that should be data loaded should be configured to use this channel.  Many times, a reload channel will 
@@ -69,6 +61,7 @@ ifdef::pro[]
 .Advanced Options
 endif::pro[]
 
+Group Link Direction:: For a node group link that is reversible, the channel can specify either "push" or "pull" to override the default group link communication.  If this field is empty, the default group link communication is used.  
 Enabled:: Indicates whether the channel is enabled or disabled.  If a channel is disabled, data is still captured for changes
 that occur on the source system, but it will not be routed and sent to the target until the channel is re-enabled.
 Reload Channel:: Indicates whether a channel is available for initial loads and reverse initial loads.

symmetric-assemble/src/asciidoc/configuration/group-links.ad

@@ -35,7 +35,7 @@ ifdef::pro[]
 endif::pro[]
 
 Sync Configuration:: Determines if configuration is also sent through this group link.  By default this is checked and configuration will communicate on this path.  There are configurations that might cause configuration to continuously loop through the network as a result this might need to be unchecked for some links.
-
+Reversible:: Allows the communication link to send in the reverse direction if specified on the channel.  A push link can be overridden to pull and a pull link can be overridden to push using a setting on the channel.
 
 .Sample Group Links
 ====

symmetric-assemble/src/asciidoc/configuration/table-triggers.ad

@@ -56,9 +56,9 @@ Sync On Insert:: Determines if changes will be captured for inserts.
 Sync On Update:: Determines if changes will be captured for updates.
 Sync On Delete:: Determines if changes will be captured for deletes.
 Reload Channel Id:: The channel_id of the channel that will be used for initial loads.
-Sync On Insert Condition:: Specify a condition for the insert trigger firing using an expression specific to the database.   This will be appended to the "WHERE" clause in a SQL expression.  
-Sync On Update Condition:: Specify a condition for the update trigger firing using an expression specific to the database.   This will be appended to the "WHERE" clause in a SQL expression.  
-Sync On Delete Condition:: Specify a condition for the delete trigger firing using an expression specific to the database.   This will be appended to the "WHERE" clause in a SQL expression.  
+Sync On Insert Condition:: Specify a condition for the insert trigger firing using an expression specific to the database.   On most platforms, it is added to an "IF" statement in the trigger text.  On SQL-Server it is added to the "WHERE" clause of a query for inserted/deleted logical tables.  
+Sync On Update Condition:: Specify a condition for the update trigger firing using an expression specific to the database.   On most platforms, it is added to an "IF" statement in the trigger text.  On SQL-Server it is added to the "WHERE" clause of a query for inserted/deleted logical tables.  
+Sync On Delete Condition:: Specify a condition for the delete trigger firing using an expression specific to the database.   On most platforms, it is added to an "IF" statement in the trigger text.  On SQL-Server it is added to the "WHERE" clause of a query for inserted/deleted logical tables.  
 Custom Insert Trigger Text:: Specify insert trigger text (SQL) to execute after the SymmetricDS trigger fires.  This field is not applicable for H2, HSQLDB 1.x or Apache Derby.
 Custom Update Trigger Text:: Specify update trigger text (SQL) to execute after the SymmetricDS trigger fires.  This field is not applicable for H2, HSQLDB 1.x or Apache Derby.
 Custom Delete Trigger Text:: Specify delete trigger text (SQL) to execute after the SymmetricDS trigger fires.  This field is not applicable for H2, HSQLDB 1.x or Apache Derby.

symmetric-assemble/src/asciidoc/configuration/transforms/types.ad

@@ -249,18 +249,18 @@ Some variables are provided to the script:
 
 |===
 
-.Tranform Expression Example Returning a String
+.Transform Expression Example Returning a String
 ====
 ----
 if (currentValue > oldValue) { 
-	return currentValue * .9 
+	return currentValue * .9;
 } else { 
-	return PRICE 
+	return PRICE; 
 }
 ----
 ====
 
-.Tranform Expression Example Returning a NewAndOldValue object
+.Transform Expression Example Returning a NewAndOldValue object
 ====
 ----
 if (currentValue != null && currentValue.length() == 0) { 
@@ -270,6 +270,22 @@ if (currentValue != null && currentValue.length() == 0) {
 }
 ----
 ====
+
+.Transform Expression Example Accessing Old/New Values for the Additional Column 'path'
+====
+----
+String newFilePath = PATH;
+String oldFilePath = null;
+if (transformedData.getOldSourceValues() != null) {
+    oldFilePath = transformedData.getOldSourceValues().get("path");
+}
+if (oldFilePath == null) {
+    return newFilePath;
+} else {
+    return oldFilePath;
+}
+----
+====
 ifndef::pro[]
 [source, SQL]
 ----

symmetric-assemble/src/asciidoc/configuration/users.ad

@@ -24,11 +24,11 @@ Authentication Method:: Method to authenticate the user.
 .Roles
 |===
 
-|Admin|Admin users can read and write data and have full control of the SymmetricDS web console.
+|Admin|Admin users have access to all functionalities of the SymmetricDS web console, including the Manage, Configure, and Explore tabs. Admin users also have full access to the connected databases.
 
-|Read|Read users can read data, but cannot write data.  Read users have a limited control of the SymmetricDS web console.
+|Read|Read users can view the Manage and Configure tabs, but have no control over editing. Read users don't have any access to the Explore tab, LDAP settings, Mail Server Settings, Monitor settings, or Notification settings. 
 
-|Write|Write users can read and write data, but have a limited control of the SymmetricDS web console. 
+|Write|Write users have access to most functionalities of the SymmetricDS web console, including the Manage, Configure, and Explore tabs. Write users also have full access to the connected databases, but do not have any access to LDAP settings, Mail Server Settings, Monitor settings, or Notification settings.
 
 |===            
 

symmetric-assemble/src/asciidoc/installation.ad

@@ -263,8 +263,11 @@ bin/sym_service stop
 
 A single SymmetricDS node may be clustered across a series of instances, creating a web farm.  A node might be clustered to provide load balancing and failover, for example.
 
-When clustered, a hardware load balancer is typically used
-to round robin client requests to the cluster.  The load balancer should be configured for stateless connections.
+When clustered, a hardware load balancer is typically used.  
+
+For clustered nodes running SymmetricDS 3.8 and later the recommended approach is to configure the load balancer to use sticky sessions or ensure the staging directory for all nodes in the cluster are using a shared network drive. 
+
+For clustered nodes running SymmetricDS 3.7 and earlier it is recommended to round robin client requests to the cluster and configure the load balancer for stateless connections.
 
 ifndef::pro[]
 Also, the `sync.url` (discussed in <<Node Properties File>>) SymmetricDS property should be set to the URL of the load balancer.

symmetric-assemble/src/asciidoc/manage/node-send.ad

@@ -3,11 +3,8 @@ Events other than data changes can be sent to nodes in the synchronization netwo
 SQL Scripts:: Sql can be sent to be executed on a target node
 BSH Scripts:: Beanshell scripts can be sent to be executed on a target node
 Table Schema:: The schema the source node can be replicated to the target node individually
-Table Data:: Tables can be loaded or reloaded individually
 
-ifdef::pro[]
-image::manage/manage-nodes-send.png[]
-endif::pro[]
+ifndef::pro[]
 
 ===== Table Data
 
@@ -21,10 +18,6 @@ if more than one table is involved, be sure to send any tables which are referre
 the channel's synchronization will block because SymmetricDS is unable to insert or update the row because the foreign key relationship refers to
 a non-existent row in the destination!
 
-ifdef::pro[]
-If you click on the *Send* button and select *Table Data* you will be prompted to select the tables you want to send.  Events will be queued up
-for each node that is selected on the Manage Nodes screen.
-endif::pro[]
 
 You can manually insert "reload" events into the <<DATA>> table that represent the table/s to reload.  These reload events are created in 
 the source database.
@@ -64,3 +57,5 @@ The following is an example insert statement:
 This insert statement generates N rows, one for each configured table that starts with sale_.  It uses the most recent
 trigger history id for the corresponding table.  It takes advantage of the initial load order for each trigger-router to
 create the three rows in the correct order (the order corresponding to the order in which the tables would have been initial loaded).
+
+endif::pro[]