From 3e2937764376e7b09dd4735358571e8923af0f33 Mon Sep 17 00:00:00 2001 From: elong Date: Mon, 23 Nov 2015 13:05:36 -0500 Subject: [PATCH 01/51] 0002443: Node communication stops running a push thread for a node --- .../symmetric/service/impl/NodeCommunicationService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeCommunicationService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeCommunicationService.java index d327655286..7aef6bb6ec 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeCommunicationService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeCommunicationService.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -29,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; @@ -76,7 +78,7 @@ public NodeCommunicationService(IClusterService clusterService, INodeService nod this.currentlyExecuting = new HashMap>(); CommunicationType[] types = CommunicationType.values(); for (CommunicationType communicationType : types) { - this.currentlyExecuting.put(communicationType, new HashSet()); + this.currentlyExecuting.put(communicationType, Collections.newSetFromMap(new ConcurrentHashMap())); } } From 4c28b7d32de6dc16dee741983ff2e79f998df1bd Mon Sep 17 00:00:00 2001 From: elong Date: Wed, 25 Nov 2015 10:44:12 -0500 Subject: [PATCH 02/51] 0002454: Use DATETIME2 in DDL when available on SQL-Server --- .../java/org/jumpmind/db/platform/mssql/MsSql2008DdlBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2008DdlBuilder.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2008DdlBuilder.java index 7ee0e6081a..ebbc79324e 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2008DdlBuilder.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2008DdlBuilder.java @@ -37,6 +37,7 @@ public MsSql2008DdlBuilder() { databaseInfo.addNativeTypeMapping(Types.DATE, "DATE", Types.DATE); databaseInfo.addNativeTypeMapping(Types.DATE, "TIME", Types.TIME); databaseInfo.addNativeTypeMapping(ColumnTypes.MSSQL_SQL_VARIANT, "SQL_VARIANT", Types.BLOB); + databaseInfo.addNativeTypeMapping(Types.TIMESTAMP, "DATETIME2"); // TODO add MSSQL 2008 types for time, datetimeoffset, and datetime2 } From 4fe95cd8faeefa627584fe76b59c82bb6e8d4e11 Mon Sep 17 00:00:00 2001 From: elong Date: Wed, 25 Nov 2015 10:50:24 -0500 Subject: [PATCH 03/51] 0002455: Use current_timestamp instead of current_date for defaults --- .../platform/mssql/MsSql2000DdlBuilder.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2000DdlBuilder.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2000DdlBuilder.java index b6694655c5..79f7a0f91e 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2000DdlBuilder.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/mssql/MsSql2000DdlBuilder.java @@ -246,14 +246,20 @@ protected String getValueAsString(Column column, Object value) { @Override protected String getNativeDefaultValue(Column column) { // Sql Server wants BIT default values as 0 or 1 - if ((column.getMappedTypeCode() == Types.BIT) - || (PlatformUtils.supportsJava14JdbcTypes() && (column.getMappedTypeCode() == PlatformUtils - .determineBooleanTypeCode()))) { - return getDefaultValueHelper().convert(column.getDefaultValue(), - column.getMappedTypeCode(), Types.SMALLINT).toString(); - } else { - return super.getNativeDefaultValue(column); - } + if ((column.getMappedTypeCode() == Types.BIT) + || (PlatformUtils.supportsJava14JdbcTypes() && (column + .getMappedTypeCode() == PlatformUtils + .determineBooleanTypeCode()))) { + return getDefaultValueHelper().convert(column.getDefaultValue(), + column.getMappedTypeCode(), Types.SMALLINT).toString(); + } + if ((column.getMappedTypeCode() == Types.TIMESTAMP) || (column.getMappedTypeCode() == Types.TIME) || (column.getMappedTypeCode() == Types.DATE)) { + String defaultValue = super.getNativeDefaultValue(column); + if (defaultValue != null && (defaultValue.equalsIgnoreCase("CURRENT_DATE") || defaultValue.equalsIgnoreCase("CURRENT DATE"))) { + return "CURRENT_TIMESTAMP"; + } + } + return super.getNativeDefaultValue(column); } @Override @@ -757,4 +763,4 @@ protected void filterColumnSqlType(StringBuilder sqlType) { sqlType.append("nvarbinary(max)"); } } -} +} \ No newline at end of file From a2d7ed93cda726b88ab2286a46302e8fc58aef3d Mon Sep 17 00:00:00 2001 From: elong Date: Wed, 25 Nov 2015 11:15:29 -0500 Subject: [PATCH 04/51] 0002456: Use unicode for bulk insert file on SQL-Server --- .../org/jumpmind/symmetric/io/MsSqlBulkDatabaseWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/MsSqlBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/MsSqlBulkDatabaseWriter.java index 27d7780fae..b2290c1c0e 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/MsSqlBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/MsSqlBulkDatabaseWriter.java @@ -216,7 +216,7 @@ protected void flush() { String sql = String.format("BULK INSERT " + this.getTargetTable().getQualifiedTableName(quote, catalogSeparator, schemaSeparator) + " FROM '" + filename) + "'" + - " WITH ( FIELDTERMINATOR='"+StringEscapeUtils.escapeJava(fieldTerminator)+"', KEEPIDENTITY" + + " WITH (DATAFILETYPE='widechar', FIELDTERMINATOR='"+StringEscapeUtils.escapeJava(fieldTerminator)+"', KEEPIDENTITY" + (fireTriggers ? ", FIRE_TRIGGERS" : "") + rowTerminatorString +");"; Statement stmt = c.createStatement(); @@ -242,4 +242,4 @@ protected void createStagingFile() { table.getName() + this.getBatch().getBatchId() + ".csv"); } -} +} \ No newline at end of file From 6d6224e7b66a35f3698063cee20c69d25205ba1a Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 30 Nov 2015 14:33:20 -0500 Subject: [PATCH 05/51] 0002433: "TIMESTAMP(6) WITH TIME ZONE" is not supported by Data Loader Type of "Oracle bulk loader." --- .../io/OracleBulkDatabaseWriter.java | 147 ++++++++++++++++-- .../io/OracleBulkDatabaseWriterTest.java | 144 ++++++++++++++++- .../test/resources/testOracleBulkWriter.xml | 22 ++- .../symmetric/io/AbstractWriterTest.java | 2 +- .../org/jumpmind/db/sql/JdbcSqlTemplate.java | 5 +- 5 files changed, 299 insertions(+), 21 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java index 08e8d9ae08..0fdbf6ae0c 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java @@ -22,20 +22,22 @@ import java.sql.Connection; import java.sql.SQLException; +import java.sql.Timestamp; import java.sql.Types; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; import java.util.Iterator; import java.util.List; - -import oracle.jdbc.OracleTypes; -import oracle.jdbc.internal.OracleCallableStatement; -import oracle.sql.ARRAY; -import oracle.sql.ArrayDescriptor; +import java.util.TimeZone; import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Column; import org.jumpmind.db.model.Table; +import org.jumpmind.db.platform.AbstractDatabasePlatform; import org.jumpmind.db.platform.IDatabasePlatform; import org.jumpmind.db.sql.BulkSqlException; import org.jumpmind.db.sql.JdbcSqlTransaction; @@ -46,6 +48,14 @@ import org.jumpmind.symmetric.io.data.writer.DefaultDatabaseWriter; import org.springframework.jdbc.support.nativejdbc.NativeJdbcExtractor; +import oracle.jdbc.OracleTypes; +import oracle.jdbc.internal.OracleCallableStatement; +import oracle.sql.ARRAY; +import oracle.sql.ArrayDescriptor; +import oracle.sql.Datum; +import oracle.sql.TIMESTAMPLTZ; +import oracle.sql.TIMESTAMPTZ; + public class OracleBulkDatabaseWriter extends DefaultDatabaseWriter { protected String procedurePrefix; @@ -82,7 +92,7 @@ public void write(CsvData data) { if (lastEventType != null && !lastEventType.equals(dataEventType)) { flush(); } - + lastEventType = dataEventType; boolean requiresFlush = false; @@ -93,6 +103,9 @@ public void write(CsvData data) { if (filterBefore(data)) { Object[] rowData = platform.getObjectValues(batch.getBinaryEncoding(), getRowData(data, CsvData.ROW_DATA), targetTable.getColumns(), false, writerSettings.isFitToColumn()); + + rowData = convertObjectValues(rowData, targetTable.getColumns()); + for (int i = 0; i < rowData.length; i++) { List columnList = null; @@ -128,10 +141,112 @@ public void write(CsvData data) { if (requiresFlush) { flush(); } - + checkForEarlyCommit(); } + /** + * @param rowData + * @param columns + * @return + */ + protected Object[] convertObjectValues(Object[] values, Column[] columns) { + if (values != null) { + for (int i = 0; i < values.length; i++) { + Object value = values[i]; + Column column = columns.length > i ? columns[i] : null; + if (column != null) { + values[i] = convertObjectValue(value, column); + } + } + } + + return values; + } + + /** + * @param value + * @param column + * @return + */ + protected Object convertObjectValue(Object value, Column column) { + if (value == null) { + return null; + } + + if (column.getMappedTypeCode() == OracleTypes.TIMESTAMPTZ + || column.getMappedTypeCode() == OracleTypes.TIMESTAMPLTZ) { + String stringValue = (String)value; + return parseTimestampTZ(column.getMappedTypeCode(), stringValue); + } + + return value; + } + + protected Datum parseTimestampTZ(int type, String value) { + Timestamp timestamp = null; + TimeZone timezone = null; + try { + // Try something like: 2015-11-20 13:37:44.000000000 + timestamp = Timestamp.valueOf(value); + timezone = TimeZone.getDefault(); + } + catch (Exception ex) { + log.debug("Failed to convert value to timestamp.", ex); + // Now expecting something like: 2015-11-20 13:37:44.000000000 +08:00 + int split = value.lastIndexOf(" "); + String timestampString = value.substring(0, split).trim(); + String timezoneString = value.substring(split).trim(); + + timestamp = Timestamp.valueOf(timestampString); + + timezone = ((AbstractDatabasePlatform)platform).getTimeZone(timezoneString); + // Even though we provide the timezone to the Oracle driver, apparently + // the timestamp component needs to actually be in UTC. + if (type == OracleTypes.TIMESTAMPTZ) { + timestamp = toUTC(timestamp, timezone); + } + } + Calendar timezoneCalender = Calendar.getInstance(); + timezoneCalender.setTimeZone(timezone); + timezoneCalender.setTime(timestamp); + + JdbcSqlTransaction jdbcTransaction = (JdbcSqlTransaction) transaction; + Connection c = jdbcTransaction.getConnection(); + try { + Connection oracleConnection = jdbcExtractor.getNativeConnection(c); + Datum ts = null; + if (type == OracleTypes.TIMESTAMPTZ) { + ts = new TIMESTAMPTZ(oracleConnection, timestamp, timezoneCalender); + } else { + ts = new TIMESTAMPLTZ(oracleConnection, timestamp); + } + return ts; + } catch (SQLException ex) { + log.info("Failed to convert '" + value + "' to TIMESTAMPTZ." ); + throw platform.getSqlTemplate().translate(ex); + } + } + + public Timestamp toUTC(Timestamp timestamp, TimeZone timezone) { + int nanos = timestamp.getNanos(); + timestamp.setNanos(0); + + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S"); + dateFormat.setTimeZone(timezone); + Date date; + try { + date = dateFormat.parse(timestamp.toString()); + } catch (ParseException ex) { + log.info("Failed to parse '" + timestamp + "'"); + throw platform.getSqlTemplate().translate(ex); + } + + Timestamp utcTimestamp = new Timestamp(date.getTime()); + utcTimestamp.setNanos(nanos); + return utcTimestamp; + } + protected void flush() { statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); try { @@ -191,7 +306,7 @@ protected void flush() { failureMessage.append(". The last flushed line number of the batch was "); failureMessage.append(statistics.get(batch).get(DataWriterStatisticConstants.LINENUMBER)); failureMessage.append("\n"); - + for (List row : rowArrays) { failureMessage.append(StringUtils.abbreviate(Arrays.toString(row.toArray(new Object[row.size()])), CsvData.MAX_DATA_SIZE_TO_PRINT_TO_LOG)); failureMessage.append("\n"); @@ -234,12 +349,12 @@ protected String getMappedType(int typeCode) { case Types.LONGVARCHAR: case Types.LONGNVARCHAR: return "varchar(4000)"; - case Types.DATE: - case Types.TIME: case OracleTypes.TIMESTAMPTZ: - return "timestamp with time zone"; + return "timestamp(9) with time zone"; case OracleTypes.TIMESTAMPLTZ: - return "timestamp with local time zone"; + return "timestamp(9) with local time zone"; + case Types.DATE: + case Types.TIME: case Types.TIMESTAMP: return "timestamp"; case Types.NUMERIC: @@ -268,10 +383,12 @@ protected String getTypeName(int typeCode) { case Types.LONGVARCHAR: case Types.LONGNVARCHAR: return String.format("%s_%s_t", procedurePrefix, "varchar").toUpperCase(); - case Types.DATE: - case Types.TIME: case OracleTypes.TIMESTAMPTZ: + return String.format("%s_%s_t", procedurePrefix, "timestamptz").toUpperCase(); case OracleTypes.TIMESTAMPLTZ: + return String.format("%s_%s_t", procedurePrefix, "timestampltz").toUpperCase(); + case Types.DATE: + case Types.TIME: case Types.TIMESTAMP: return String.format("%s_%s_t", procedurePrefix, "timestamp").toUpperCase(); case Types.NUMERIC: @@ -284,7 +401,7 @@ protected String getTypeName(int typeCode) { case Types.INTEGER: return String.format("%s_%s_t", procedurePrefix, "integer").toUpperCase(); default: - throw new UnsupportedOperationException(Integer.toString(typeCode)); + throw new UnsupportedOperationException("OracleBulkDatabaseWriter does not support type: " + Integer.toString(typeCode)); } } diff --git a/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java b/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java index bdaad2a05d..98a20775bf 100644 --- a/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java +++ b/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java @@ -20,8 +20,13 @@ */ package org.jumpmind.symmetric.io; +import java.sql.Connection; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; import org.jumpmind.db.DbTestUtils; import org.jumpmind.db.platform.oracle.OracleDatabasePlatform; @@ -33,6 +38,10 @@ import org.junit.BeforeClass; import org.junit.Test; import org.springframework.jdbc.support.nativejdbc.CommonsDbcpNativeJdbcExtractor; +import org.springframework.jdbc.support.nativejdbc.NativeJdbcExtractor; + +import oracle.sql.TIMESTAMPLTZ; +import oracle.sql.TIMESTAMPTZ; public class OracleBulkDatabaseWriterTest extends AbstractWriterTest { @@ -83,7 +92,133 @@ public void testInsert1000Rows() { Assert.assertEquals(count, countRows("test_bulkload_table_1")); } + } + + @Test + public void testInsertTimestampTZ_timestamp() throws Exception { + if (platform != null && platform instanceof OracleDatabasePlatform) { + + NativeJdbcExtractor jdbcExtractor = new CommonsDbcpNativeJdbcExtractor(); + + platform.getSqlTemplate().update("truncate table test_bulkload_table_1"); + List datas = new ArrayList(); + + String id = getNextId(); + + String[] values = { id, "string2", "string not null2", "char2", + "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", + "47", "67.89", "-0.0747663", "2007-01-02 03:20:10.000", "2007-01-02 03:20:10.000", + "2007-01-02 03:20:10.000", "2007-01-02 03:20:10.000" }; + CsvData data = new CsvData(DataEventType.INSERT, values); + datas.add(data); + + long count = writeData(new TableCsvData(platform.getTableFromCache( + "test_bulkload_table_1", false), datas)); + + Map rowData = queryForRow(id); + DataSource datasource = (DataSource)platform.getDataSource(); + Connection connection = datasource.getConnection(); + Connection oracleConnection = jdbcExtractor.getNativeConnection(connection); + + final String EXPECTED_TIMESTAMPTZ = "2007-01-02 03:20:10.0 America/New_York"; + + checkTimestampTZ(rowData.get("TIMESTAMPTZ0_VALUE"), oracleConnection, EXPECTED_TIMESTAMPTZ); + checkTimestampTZ(rowData.get("TIMESTAMPTZ3_VALUE"), oracleConnection, EXPECTED_TIMESTAMPTZ); + checkTimestampTZ(rowData.get("TIMESTAMPTZ6_VALUE"), oracleConnection, EXPECTED_TIMESTAMPTZ); + checkTimestampTZ(rowData.get("TIMESTAMPTZ9_VALUE"), oracleConnection, EXPECTED_TIMESTAMPTZ); + + Assert.assertEquals(count, countRows("test_bulkload_table_1")); + } + } + + @Test + public void testInsertTimestampTZ_timestampWithTimeZone() throws Exception { + if (platform != null && platform instanceof OracleDatabasePlatform) { + + NativeJdbcExtractor jdbcExtractor = new CommonsDbcpNativeJdbcExtractor(); + + platform.getSqlTemplate().update("truncate table test_bulkload_table_1"); + + List datas = new ArrayList(); + + String id = getNextId(); + + String[] values = { id, "string2", "string not null2", "char2", + "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", + "47", "67.89", "-0.0747663", "2007-01-02 03:20:10.123456789 -08:00", "2007-01-02 03:20:10.123456789 -08:00", + "2007-01-02 03:20:10.123456789 -08:00", "2007-01-02 03:20:10.123456789 -08:00" }; + CsvData data = new CsvData(DataEventType.INSERT, values); + datas.add(data); + + long count = writeData(new TableCsvData(platform.getTableFromCache( + "test_bulkload_table_1", false), datas)); + + Map rowData = queryForRow(id); + DataSource datasource = (DataSource)platform.getDataSource(); + Connection connection = datasource.getConnection(); + Connection oracleConnection = jdbcExtractor.getNativeConnection(connection); + + checkTimestampTZ(rowData.get("TIMESTAMPTZ0_VALUE"), oracleConnection, "2007-01-02 03:20:10.0 -8:00"); + checkTimestampTZ(rowData.get("TIMESTAMPTZ3_VALUE"), oracleConnection, "2007-01-02 03:20:10.123 -8:00"); + checkTimestampTZ(rowData.get("TIMESTAMPTZ6_VALUE"), oracleConnection, "2007-01-02 03:20:10.123457 -8:00"); + checkTimestampTZ(rowData.get("TIMESTAMPTZ9_VALUE"), oracleConnection, "2007-01-02 03:20:10.123456789 -8:00"); + + Assert.assertEquals(count, countRows("test_bulkload_table_1")); + } + } + + @Test + public void testInsertTimestampTZ_timestampWithLocalTimeZone() throws Exception { + if (platform != null && platform instanceof OracleDatabasePlatform) { + + NativeJdbcExtractor jdbcExtractor = new CommonsDbcpNativeJdbcExtractor(); + + platform.getSqlTemplate().update("truncate table test_bulkload_table_1"); + + List datas = new ArrayList(); + + String id = getNextId(); + + String[] values = { id, "string2", "string not null2", "char2", + "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", + "47", "67.89", "-0.0747663", null, null, + null, null, "2007-01-02 03:20:10.123456789 -08:00" }; + CsvData data = new CsvData(DataEventType.INSERT, values); + datas.add(data); + + long count = writeData(new TableCsvData(platform.getTableFromCache( + "test_bulkload_table_1", false), datas)); + + Map rowData = queryForRow(id); + DataSource datasource = (DataSource)platform.getDataSource(); + Connection connection = datasource.getConnection(); + Connection oracleConnection = jdbcExtractor.getNativeConnection(connection); + + checkTimestampLTZ(rowData.get("TIMESTAMPLTZ9_VALUE"), oracleConnection, "2007-01-02 03:20:10.123456789 America/New_York"); + + Assert.assertEquals(count, countRows("test_bulkload_table_1")); + } + } + + /** + * @param rowData + * @param oracleConnection + * @param EXPECTED_TIMESTAMPTZ + * @throws SQLException + */ + private void checkTimestampTZ(Object value, Connection oracleConnection, final String EXPECTED_TIMESTAMPTZ) + throws SQLException { + TIMESTAMPTZ timestamp = (TIMESTAMPTZ) value; + String actualTimestampString = timestamp.stringValue(oracleConnection); + Assert.assertEquals(EXPECTED_TIMESTAMPTZ, actualTimestampString); + } + + private void checkTimestampLTZ(Object value, Connection oracleConnection, final String EXPECTED_TIMESTAMPTZ) + throws SQLException { + TIMESTAMPLTZ timestamp = (TIMESTAMPLTZ) value; + String actualTimestampString = timestamp.stringValue(oracleConnection); + Assert.assertEquals(EXPECTED_TIMESTAMPTZ, actualTimestampString); } @Test @@ -91,20 +226,24 @@ public void testInsertCollision() { if (platform != null && platform instanceof OracleDatabasePlatform) { platform.getSqlTemplate().update("truncate table test_bulkload_table_1"); - String[] values = { getNextId(), "string2", "string not null2", "char2", + String id = getNextId(); + + String[] values = { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663" }; CsvData data = new CsvData(DataEventType.INSERT, values); writeData(data, values); Assert.assertEquals(1, countRows("test_bulkload_table_1")); + try { setErrorExpected(true); List datas = new ArrayList(); datas.add(data); + for (int i = 0; i < 10; i++) { - values = new String[] { getNextId(), "string2", "string not null2", "char2", + values = new String[] { id, "string2", "string not null2", "char2", "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", "47", "67.89", "-0.0747663" }; data = new CsvData(DataEventType.INSERT, values); @@ -121,6 +260,5 @@ public void testInsertCollision() { setErrorExpected(false); } } - } } diff --git a/symmetric-client/src/test/resources/testOracleBulkWriter.xml b/symmetric-client/src/test/resources/testOracleBulkWriter.xml index 69edb96ce0..a3c8c0e1c0 100644 --- a/symmetric-client/src/test/resources/testOracleBulkWriter.xml +++ b/symmetric-client/src/test/resources/testOracleBulkWriter.xml @@ -21,7 +21,7 @@ under the License. --> - + @@ -40,6 +40,26 @@ + + + + + + + + + + + + + + + + + + diff --git a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/AbstractWriterTest.java b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/AbstractWriterTest.java index 06f24133e9..c7de369d0d 100644 --- a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/AbstractWriterTest.java +++ b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/AbstractWriterTest.java @@ -276,7 +276,7 @@ public boolean isErrorExpected() { } public Map queryForRow(String id) { - return platform.getSqlTemplate().queryForMap("select * from " + TEST_TABLE + " where id=?", new Integer(id)); + return platform.getSqlTemplate().queryForMap("select * from " + getTestTable() + " where id=?", new Integer(id)); } protected class TableCsvData { diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java index 7dd413d47d..17c1041bc4 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java @@ -268,7 +268,10 @@ public Map execute(Connection con) throws SQLException { && superClazz.getName().equals("oracle.sql.Datum")) { try { Method method = superClazz.getMethod("toJdbc"); - value = method.invoke(value); + Object jdbcValue = method.invoke(value); + if (jdbcValue != null) { // Oracle TIMESTAMPTZ (for example) will not convert through toJdbc. + value = jdbcValue; + } } catch (Exception e) { throw new IllegalStateException(e); } From 7919469eb71912c615d5f9951e06882eb76756cf Mon Sep 17 00:00:00 2001 From: mmichalek Date: Wed, 2 Dec 2015 11:10:17 -0500 Subject: [PATCH 06/51] 0002459: Timestamp Values may get exported with invalid format --- .../jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java index 0cf84c1382..6166fa88da 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java @@ -36,8 +36,8 @@ public OracleTriggerTemplate(ISymmetricDialect symmetricDialect) { geometryColumnTemplate = "case when $(tableAlias).\"$(columnName)\" is null then to_clob('') else '\"'||replace(replace(SDO_UTIL.TO_WKTGEOMETRY($(tableAlias).\"$(columnName)\"),'\\','\\\\'),'\"','\\\"')||'\"' end"; numberColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', '\"'||cast($(tableAlias).\"$(columnName)\" as number("+symmetricDialect.getTemplateNumberPrecisionSpec()+"))||'\"')" ; datetimeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF3')),'\"'))" ; - dateTimeWithTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')),'\"'))" ; - dateTimeWithLocalTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF')),'\"'))" ; + dateTimeWithTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM')),'\"'))" ; + dateTimeWithLocalTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; timeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; dateColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; clobColumnTemplate = "decode(dbms_lob.getlength($(tableAlias).\"$(columnName)\"), null, to_clob(''), '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"')" ; From fada6a48fd1d81f62b71b9a3075ff41a5dbbdb12 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Wed, 2 Dec 2015 11:17:39 -0500 Subject: [PATCH 07/51] 0002433: "TIMESTAMP(6) WITH TIME ZONE" is not supported by Data Loader Type of "Oracle bulk loader." --- .../io/OracleBulkDatabaseWriter.java | 76 ++++++++++--------- .../io/OracleBulkDatabaseWriterTest.java | 38 ++++++++++ 2 files changed, 80 insertions(+), 34 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java index 0fdbf6ae0c..2ddeffb553 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriter.java @@ -179,41 +179,49 @@ protected Object convertObjectValue(Object value, Column column) { String stringValue = (String)value; return parseTimestampTZ(column.getMappedTypeCode(), stringValue); } - + return value; } protected Datum parseTimestampTZ(int type, String value) { - Timestamp timestamp = null; - TimeZone timezone = null; - try { - // Try something like: 2015-11-20 13:37:44.000000000 - timestamp = Timestamp.valueOf(value); - timezone = TimeZone.getDefault(); - } - catch (Exception ex) { - log.debug("Failed to convert value to timestamp.", ex); - // Now expecting something like: 2015-11-20 13:37:44.000000000 +08:00 - int split = value.lastIndexOf(" "); - String timestampString = value.substring(0, split).trim(); - String timezoneString = value.substring(split).trim(); - - timestamp = Timestamp.valueOf(timestampString); - - timezone = ((AbstractDatabasePlatform)platform).getTimeZone(timezoneString); - // Even though we provide the timezone to the Oracle driver, apparently - // the timestamp component needs to actually be in UTC. - if (type == OracleTypes.TIMESTAMPTZ) { - timestamp = toUTC(timestamp, timezone); - } + if (value == null || StringUtils.isEmpty(value.trim())) { + return null; } - Calendar timezoneCalender = Calendar.getInstance(); - timezoneCalender.setTimeZone(timezone); - timezoneCalender.setTime(timestamp); - JdbcSqlTransaction jdbcTransaction = (JdbcSqlTransaction) transaction; - Connection c = jdbcTransaction.getConnection(); try { + Timestamp timestamp = null; + TimeZone timezone = null; + try { + // Try something like: 2015-11-20 13:37:44.000000000 + timestamp = Timestamp.valueOf(value.trim()); + timezone = TimeZone.getDefault(); + } + catch (Exception ex) { + log.debug("Failed to convert value to timestamp.", ex); + // Now expecting something like: 2015-11-20 13:37:44.000000000 +08:00 + int split = value.lastIndexOf(" "); + String timestampString = value.substring(0, split).trim(); + if (timestampString.endsWith(".")) { // Cover case where triggers would export format like "2007-01-02 03:20:10." + timestampString = timestampString.substring(0, timestampString.length()-1); + } + String timezoneString = value.substring(split).trim(); + + timestamp = Timestamp.valueOf(timestampString); + + timezone = ((AbstractDatabasePlatform)platform).getTimeZone(timezoneString); + // Even though we provide the timezone to the Oracle driver, apparently + // the timestamp component needs to actually be in UTC. + if (type == OracleTypes.TIMESTAMPTZ) { + timestamp = toUTC(timestamp, timezone); + } + } + Calendar timezoneCalender = Calendar.getInstance(); + timezoneCalender.setTimeZone(timezone); + timezoneCalender.setTime(timestamp); + + JdbcSqlTransaction jdbcTransaction = (JdbcSqlTransaction) transaction; + Connection c = jdbcTransaction.getConnection(); + Connection oracleConnection = jdbcExtractor.getNativeConnection(c); Datum ts = null; if (type == OracleTypes.TIMESTAMPTZ) { @@ -222,16 +230,16 @@ protected Datum parseTimestampTZ(int type, String value) { ts = new TIMESTAMPLTZ(oracleConnection, timestamp); } return ts; - } catch (SQLException ex) { + } catch (Exception ex) { log.info("Failed to convert '" + value + "' to TIMESTAMPTZ." ); throw platform.getSqlTemplate().translate(ex); } } - + public Timestamp toUTC(Timestamp timestamp, TimeZone timezone) { int nanos = timestamp.getNanos(); timestamp.setNanos(0); - + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S"); dateFormat.setTimeZone(timezone); Date date; @@ -241,7 +249,7 @@ public Timestamp toUTC(Timestamp timestamp, TimeZone timezone) { log.info("Failed to parse '" + timestamp + "'"); throw platform.getSqlTemplate().translate(ex); } - + Timestamp utcTimestamp = new Timestamp(date.getTime()); utcTimestamp.setNanos(nanos); return utcTimestamp; @@ -467,7 +475,7 @@ protected void buildBulkInsertProcedure(Table table) { ddl.append(String.format("%s(i), \n", variable)); } ddl.replace(ddl.length()-3, ddl.length(), ");\n"); - + ddl.append(String.format("exception \n")); ddl.append(String.format(" when dml_errors then \n")); ddl.append(String.format(" for i in 1 .. SQL%%BULK_EXCEPTIONS.count loop \n")); @@ -478,7 +486,7 @@ protected void buildBulkInsertProcedure(Table table) { ddl.append(String.format(" o_errors(o_errors.count) := SQL%%BULK_EXCEPTIONS(i).ERROR_INDEX; \n")); ddl.append(String.format(" end loop; \n")); ddl.append(String.format("end %s; ", procedureName)); - + if (log.isDebugEnabled()) { log.debug(ddl.toString()); } diff --git a/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java b/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java index 98a20775bf..4ac1f662f8 100644 --- a/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java +++ b/symmetric-client/src/test/java/org/jumpmind/symmetric/io/OracleBulkDatabaseWriterTest.java @@ -168,6 +168,44 @@ public void testInsertTimestampTZ_timestampWithTimeZone() throws Exception { } } + @Test + public void testInsertTimestampTZ_timestampWithTimeZoneNull() throws Exception { + if (platform != null && platform instanceof OracleDatabasePlatform) { + + NativeJdbcExtractor jdbcExtractor = new CommonsDbcpNativeJdbcExtractor(); + + platform.getSqlTemplate().update("truncate table test_bulkload_table_1"); + + List datas = new ArrayList(); + + String id = getNextId(); + + String[] values = { id, "string2", "string not null2", "char2", + "char not null2", "2007-01-02 03:20:10.000", "2007-02-03 04:05:06.000", "0", + "47", "67.89", "-0.0747663", "2007-01-02 03:20:10. -08:00", + "", + " ", + null }; + CsvData data = new CsvData(DataEventType.INSERT, values); + datas.add(data); + + long count = writeData(new TableCsvData(platform.getTableFromCache( + "test_bulkload_table_1", false), datas)); + + Map rowData = queryForRow(id); + DataSource datasource = (DataSource)platform.getDataSource(); + Connection connection = datasource.getConnection(); + Connection oracleConnection = jdbcExtractor.getNativeConnection(connection); + + checkTimestampTZ(rowData.get("TIMESTAMPTZ0_VALUE"), oracleConnection, "2007-01-02 03:20:10.0 -8:00"); + Assert.assertNull(rowData.get("TIMESTAMPTZ3_VALUE")); + Assert.assertNull(rowData.get("TIMESTAMPTZ6_VALUE")); + Assert.assertNull(rowData.get("TIMESTAMPTZ9_VALUE")); + + Assert.assertEquals(count, countRows("test_bulkload_table_1")); + } + } + @Test public void testInsertTimestampTZ_timestampWithLocalTimeZone() throws Exception { if (platform != null && platform instanceof OracleDatabasePlatform) { From 6a9003b8b1a32d9042724054ac2053954457757b Mon Sep 17 00:00:00 2001 From: elong Date: Wed, 2 Dec 2015 15:32:20 -0500 Subject: [PATCH 08/51] fix delete event processing --- symmetric-client-clib/inc/model/Node.h | 2 +- symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-client-clib/inc/model/Node.h b/symmetric-client-clib/inc/model/Node.h index 6d9001ca91..c5d3a50d01 100644 --- a/symmetric-client-clib/inc/model/Node.h +++ b/symmetric-client-clib/inc/model/Node.h @@ -23,7 +23,7 @@ #include -#define SYM_VERSION "3.7.23" +#define SYM_VERSION "3.7.27" typedef enum SymNodeStatus { SYM_NODE_STATUS_DATA_LOAD_NOT_STARTED, diff --git a/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c b/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c index 7ba535c7aa..ee5e54d0bf 100644 --- a/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c +++ b/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c @@ -161,7 +161,7 @@ int SymDefaultDatabaseWriter_delete(SymDefaultDatabaseWriter *this, SymCsvData * this->dmlStatement->destroy(this->dmlStatement); } // TODO: pass nullKeyIndiciators - this->dmlStatement = SymDmlStatement_new(NULL, SYM_DML_TYPE_UPDATE, this->targetTable, NULL, &this->platform->databaseInfo); + this->dmlStatement = SymDmlStatement_new(NULL, SYM_DML_TYPE_DELETE, this->targetTable, NULL, &this->platform->databaseInfo); this->sqlTransaction->prepare(this->sqlTransaction, this->dmlStatement->sql, &error); this->isError = error != 0; } From 512558e8aa835765b03a1a16aceead420b7508db Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Thu, 3 Dec 2015 15:33:22 -0500 Subject: [PATCH 09/51] 0002460 - Support for redshift driver --- .../platform/JdbcDatabasePlatformFactory.java | 4 +++ .../platform/redshift/RedshiftDdlReader.java | 27 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java index 3e522e8da6..5147b8c85b 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java @@ -302,11 +302,15 @@ private static boolean isRedshiftDatabase(Connection connection) { try { DatabaseMetaData dmd = connection.getMetaData(); dmd.getMaxColumnsInIndex(); + if (dmd.getDriverName().toUpperCase().contains("REDSHIFT")) { + isRedshift = true; + } } catch (SQLException ex) { if (ex.getSQLState().equals("99999")) { isRedshift = true; } } + return isRedshift; } diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlReader.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlReader.java index ce1b594eae..3f31331340 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlReader.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlReader.java @@ -20,11 +20,13 @@ */ package org.jumpmind.db.platform.redshift; +import java.sql.Connection; import java.sql.SQLException; import java.sql.Types; import java.util.Map; import org.jumpmind.db.model.Column; +import org.jumpmind.db.model.Table; import org.jumpmind.db.model.TypeMap; import org.jumpmind.db.platform.AbstractJdbcDdlReader; import org.jumpmind.db.platform.DatabaseMetaDataWrapper; @@ -39,6 +41,17 @@ public RedshiftDdlReader(IDatabasePlatform platform) { setDefaultTablePattern(null); } + @Override + protected Table readTable(Connection connection, DatabaseMetaDataWrapper metaData, Map values) + throws SQLException { + Table table = super.readTable(connection, metaData, values); + + if (table != null) { + determineAutoIncrementFromResultSetMetaData(connection, table, table.getColumns()); + } + return table; + } + @Override protected Column readColumn(DatabaseMetaDataWrapper metaData, Map values) throws SQLException { Column column = super.readColumn(metaData, values); @@ -111,6 +124,9 @@ private String extractDelimitedDefaultValue(String defaultValue) { * Extracts the default value from a default value spec of the form * "-9000000000000000000::bigint". * + * Handles sequence evaluations where the end paren is after the :: + * nextval('"sym_data_data_id_seq"'::text) + * * @param defaultValue The default value spec * * @return The default value @@ -119,7 +135,16 @@ private String extractUndelimitedDefaultValue(String defaultValue) { int valueEnd = defaultValue.indexOf("::"); if (valueEnd > 0) { - defaultValue = defaultValue.substring(0, valueEnd); + + defaultValue = defaultValue.substring(0, valueEnd); + + int startParen = defaultValue.indexOf("("); + int endParen = defaultValue.indexOf(")"); + + if (startParen > 0 && endParen < 0) { + defaultValue = defaultValue + ")"; + } + } else { if (defaultValue.startsWith("(") && defaultValue.endsWith(")")) { defaultValue = defaultValue.substring(1, defaultValue.length() - 1); From 24de68a87be8462ad7cd55915f84cdf70ed56280 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 7 Dec 2015 11:00:49 -0500 Subject: [PATCH 10/51] Fix crash when processing incoming offline file. (Remove incorrect/unexpected destroy.) --- symmetric-client-clib/src/transport/file/FileIncomingTransport.c | 1 - 1 file changed, 1 deletion(-) diff --git a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c index c0894c0116..051bbab597 100644 --- a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c @@ -122,7 +122,6 @@ long SymFileIncomingTransport_process(SymFileIncomingTransport *this, SymDataPro } void SymFileIncomingTransport_destroy(SymFileIncomingTransport *this) { - free(this->remoteNode); free(this); } From bf5f5d18a143f65e79cb0b088e2cefd1666c7613 Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Mon, 7 Dec 2015 20:08:16 -0500 Subject: [PATCH 11/51] 0002462: Replication does not restart after network reconnection between servers --- .../db/platform/postgresql/PostgreSqlJdbcSqlTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlJdbcSqlTemplate.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlJdbcSqlTemplate.java index 692b8ef524..5449e694dd 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlJdbcSqlTemplate.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlJdbcSqlTemplate.java @@ -35,7 +35,7 @@ public PostgreSqlJdbcSqlTemplate(DataSource dataSource, SqlTemplateSettings sett super(dataSource, settings, lobHandler, databaseInfo); this.requiresAutoCommitFalseToSetFetchSize = true; primaryKeyViolationSqlStates = new String[] { "23000", "23505" }; - primaryKeyViolationMessageParts = new String[] {"duplicate key value violates"}; + primaryKeyViolationMessageParts = new String[] {"duplicate key value violates", "duplicar valor da chave viola a restrição de unicidade"}; foreignKeyViolationMessageParts = new String[] {"violates foreign key constraint"}; } From 4de0f9c9c67279af844540ab7057777c41fa3c97 Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Tue, 8 Dec 2015 07:11:24 -0500 Subject: [PATCH 12/51] 0002463 - Redshift convert clobs to max varchar --- .../org/jumpmind/db/platform/redshift/RedshiftDdlBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlBuilder.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlBuilder.java index eb69ba1a0d..626cf92d43 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlBuilder.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/redshift/RedshiftDdlBuilder.java @@ -46,7 +46,8 @@ public RedshiftDdlBuilder() { databaseInfo.addNativeTypeMapping(Types.LONGVARCHAR, "VARCHAR(65535)"); databaseInfo.addNativeTypeMapping(Types.TINYINT, "SMALLINT", Types.SMALLINT); databaseInfo.addNativeTypeMapping(Types.TIME, "TIMESTAMP", Types.TIMESTAMP); - + databaseInfo.addNativeTypeMapping(Types.CLOB, "VARCHAR(65535)"); + databaseInfo.setDefaultSize(Types.CHAR, 256); databaseInfo.setDefaultSize(Types.VARCHAR, 256); From de4b6c6f736f11edc460e12affe114c9bed0aa2d Mon Sep 17 00:00:00 2001 From: "Hicks, Josh" Date: Tue, 8 Dec 2015 07:28:24 -0500 Subject: [PATCH 13/51] 0002460 - Redshift driver support --- .../main/java/org/jumpmind/db/platform/DdlBuilderFactory.java | 3 +++ .../src/main/java/org/jumpmind/symmetric/io/data/DbExport.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/DdlBuilderFactory.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/DdlBuilderFactory.java index 66d1c73f4b..d445c0e567 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/DdlBuilderFactory.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/DdlBuilderFactory.java @@ -37,6 +37,7 @@ import org.jumpmind.db.platform.mysql.MySqlDdlBuilder; import org.jumpmind.db.platform.oracle.OracleDdlBuilder; import org.jumpmind.db.platform.postgresql.PostgreSqlDdlBuilder; +import org.jumpmind.db.platform.redshift.RedshiftDdlBuilder; import org.jumpmind.db.platform.sqlanywhere.SqlAnywhereDdlBuilder; import org.jumpmind.db.platform.sqlite.SqliteDdlBuilder; @@ -91,6 +92,8 @@ public static final IDdlBuilder createDdlBuilder(String databaseName) { return new AseDdlBuilder(); } else if (DatabaseNamesConstants.SQLANYWHERE.equalsIgnoreCase(databaseName)) { return new SqlAnywhereDdlBuilder(); + } else if (DatabaseNamesConstants.REDSHIFT.equalsIgnoreCase(databaseName)) { + return new RedshiftDdlBuilder(); } else { return null; } diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/DbExport.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/DbExport.java index 85b1599094..d18699d583 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/DbExport.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/DbExport.java @@ -64,7 +64,7 @@ public enum Format { }; public enum Compatible { - DB2, DERBY, FIREBIRD, FIREBIRD_DIALECT1, GREENPLUM, H2, HSQLDB, HSQLDB2, INFORMIX, INTERBASE, MSSQL, MSSQL2000, MSSQL2005, MSSQL2008, MYSQL, ORACLE, POSTGRES, SYBASE, SQLITE, MARIADB, ASE, SQLANYWHERE + DB2, DERBY, FIREBIRD, FIREBIRD_DIALECT1, GREENPLUM, H2, HSQLDB, HSQLDB2, INFORMIX, INTERBASE, MSSQL, MSSQL2000, MSSQL2005, MSSQL2008, MYSQL, ORACLE, POSTGRES, SYBASE, SQLITE, MARIADB, ASE, SQLANYWHERE, REDSHIFT }; private Format format = Format.SQL; From 366c4876c4a6fd5f2ba1c99b4f73d3327ad505d0 Mon Sep 17 00:00:00 2001 From: elong Date: Tue, 8 Dec 2015 09:15:51 -0500 Subject: [PATCH 14/51] fix debug logging --- symmetric-client-clib/src/transport/http/HttpTransportManager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/src/transport/http/HttpTransportManager.c b/symmetric-client-clib/src/transport/http/HttpTransportManager.c index aa522497ad..947e920149 100644 --- a/symmetric-client-clib/src/transport/http/HttpTransportManager.c +++ b/symmetric-client-clib/src/transport/http/HttpTransportManager.c @@ -88,7 +88,7 @@ static char * getAcknowledgementData(SymList *batches) { static int sendMessage(SymHttpTransportManager *this, char *url, char *postData) { long httpResponseCode = -1; - SymLog_info("Sending message '%s' to URL '%s'", postData, url); + SymLog_debug("Sending message '%s' to URL '%s'", postData, url); CURL *curl = curl_easy_init(); if (curl) { if (this->parameterService->is(this->parameterService, SYM_PARAMETER_HTTPS_VERIFIED_SERVERS, 1)) { From b0f089ece587199e83f4db042e9409915bb3003c Mon Sep 17 00:00:00 2001 From: elong Date: Tue, 8 Dec 2015 09:16:08 -0500 Subject: [PATCH 15/51] fix debug logging --- .../src/db/platform/sqlite/SqliteSqlTemplate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/src/db/platform/sqlite/SqliteSqlTemplate.c b/symmetric-client-clib/src/db/platform/sqlite/SqliteSqlTemplate.c index a5d80fae7d..3818cd2b40 100644 --- a/symmetric-client-clib/src/db/platform/sqlite/SqliteSqlTemplate.c +++ b/symmetric-client-clib/src/db/platform/sqlite/SqliteSqlTemplate.c @@ -22,7 +22,7 @@ #include "common/Log.h" static void SymSqliteSqlTemplate_prepare(SymSqliteSqlTemplate *this, char *sql, SymStringArray *args, SymList *sqlTypes, int *error, sqlite3_stmt **stmt) { - SymLog_info("Preparing %s", sql); + SymLog_debug("Preparing %s", sql); int rc = sqlite3_prepare_v2(this->db, sql, -1, stmt, NULL); if (rc != SQLITE_OK) { SymLog_error("Failed to prepare query: %s", sql); From a8e1357f8dfc615ee04205c366f340945af60cf2 Mon Sep 17 00:00:00 2001 From: elong Date: Tue, 8 Dec 2015 11:18:59 -0500 Subject: [PATCH 16/51] fix debug logging --- symmetric-client-clib/src/core/JobManager.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/symmetric-client-clib/src/core/JobManager.c b/symmetric-client-clib/src/core/JobManager.c index f2ef1d788e..c3575b38e1 100644 --- a/symmetric-client-clib/src/core/JobManager.c +++ b/symmetric-client-clib/src/core/JobManager.c @@ -42,22 +42,23 @@ unsigned short SymJobManager_shouldRun(SymJobManager *this, char* startJobProper void SymJobManager_invoke(SymJobManager *this) { if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_SYNCTRIGGERS_JOB, SYM_PARAMETER_SYNCTRIGGERS_PERIOD_MS, this->lastSyncTriggersTime)) { - SymLog_info("SYNC TRIGGERS ============================)"); + SymLog_debug("SYNC TRIGGERS ============================"); this->engine->syncTriggers(this->engine); time(&this->lastSyncTriggersTime); } if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_ROUTE_JOB, SYM_PARAMETER_ROUTE_PERIOD_MS, this->lastRouteTime)) { - SymLog_info("ROUTE ============================)"); + SymLog_debug("ROUTE ============================"); this->engine->route(this->engine); time(&this->lastRouteTime); } if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_PUSH_JOB, SYM_PARAMETER_PUSH_PERIOD_MS, this->lastPushTime)) { + SymLog_debug("PUSH ============================"); SymRemoteNodeStatuses *pushStatus = this->engine->push(this->engine); if (pushStatus->wasBatchProcessed(pushStatus) || pushStatus->wasDataProcessed(pushStatus)) { // Only run heartbeat after a successful push to avoid queueing up lots of offline heartbeats. if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_HEARTBEAT_JOB, SYM_PARAMETER_HEARTBEAT_JOB_PERIOD_MS, this->lastHeartbeatTime)) { - SymLog_info("HEARTBEAT ============================)"); + SymLog_debug("HEARTBEAT ============================"); this->engine->heartbeat(this->engine, 0); time(&this->lastHeartbeatTime); } @@ -66,24 +67,24 @@ void SymJobManager_invoke(SymJobManager *this) { pushStatus->destroy(pushStatus); } if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_PULL_JOB, SYM_PARAMETER_PULL_PERIOD_MS, this->lastPullTime)) { - SymLog_info("PULL ============================)"); + SymLog_debug("PULL ============================"); SymRemoteNodeStatuses *statuses = this->engine->pull(this->engine); time(&this->lastPullTime); statuses->destroy(statuses); } if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_PURGE_JOB, SYM_PARAMETER_PURGE_PERIOD_MS, this->lastPurgeTime)) { - SymLog_info("PURGE ============================)"); + SymLog_debug("PURGE ============================"); this->engine->purge(this->engine); time(&this->lastPurgeTime); } if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_OFFLINE_PUSH_JOB, SYM_PARAMETER_OFFLINE_PUSH_PERIOD_MS, this->lastOfflinePushTime)) { - SymLog_info("OFFLINE PUSH ============================)"); + SymLog_debug("OFFLINE PUSH ============================"); SymRemoteNodeStatuses *statuses = this->engine->offlinePushService->pushData(this->engine->offlinePushService); time(&this->lastOfflinePushTime); statuses->destroy(statuses); } if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_OFFLINE_PULL_JOB, SYM_PARAMETER_OFFLINE_PULL_PERIOD_MS, this->lastOfflinePullTime)) { - SymLog_info("OFFLINE PULL ============================)"); + SymLog_debug("OFFLINE PULL ============================"); SymRemoteNodeStatuses *statuses = this->engine->offlinePullService->pullData(this->engine->offlinePullService); time(&this->lastOfflinePullTime); statuses->destroy(statuses); From 833f6e644604d38c52f40b5593f4c0f9d9be9825 Mon Sep 17 00:00:00 2001 From: elong Date: Tue, 8 Dec 2015 11:31:06 -0500 Subject: [PATCH 17/51] heartbeat can run if push is not failing --- symmetric-client-clib/src/core/JobManager.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/symmetric-client-clib/src/core/JobManager.c b/symmetric-client-clib/src/core/JobManager.c index c3575b38e1..823ba3a18b 100644 --- a/symmetric-client-clib/src/core/JobManager.c +++ b/symmetric-client-clib/src/core/JobManager.c @@ -54,8 +54,7 @@ void SymJobManager_invoke(SymJobManager *this) { if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_PUSH_JOB, SYM_PARAMETER_PUSH_PERIOD_MS, this->lastPushTime)) { SymLog_debug("PUSH ============================"); SymRemoteNodeStatuses *pushStatus = this->engine->push(this->engine); - if (pushStatus->wasBatchProcessed(pushStatus) - || pushStatus->wasDataProcessed(pushStatus)) { + if (pushStatus->errorOccurred(pushStatus) == 0) { // Only run heartbeat after a successful push to avoid queueing up lots of offline heartbeats. if (SymJobManager_shouldRun(this, SYM_PARAMETER_START_HEARTBEAT_JOB, SYM_PARAMETER_HEARTBEAT_JOB_PERIOD_MS, this->lastHeartbeatTime)) { SymLog_debug("HEARTBEAT ============================"); From eca72a106293773beb0d29a0540f5615e0de5349 Mon Sep 17 00:00:00 2001 From: elong Date: Tue, 8 Dec 2015 11:31:51 -0500 Subject: [PATCH 18/51] add sqlite.init.sql for running pragma on database --- .../inc/common/ParameterConstants.h | 1 + .../inc/db/platform/sqlite/SqlitePlatform.h | 1 + symmetric-client-clib/inc/model/Node.h | 2 +- .../src/db/platform/sqlite/SqlitePlatform.c | 15 ++++++++++++++- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/symmetric-client-clib/inc/common/ParameterConstants.h b/symmetric-client-clib/inc/common/ParameterConstants.h index a29bff4137..fb9dd85098 100644 --- a/symmetric-client-clib/inc/common/ParameterConstants.h +++ b/symmetric-client-clib/inc/common/ParameterConstants.h @@ -94,5 +94,6 @@ #define SYM_PARAMETER_HTTPS_ALLOW_SELF_SIGNED_CERTS "https.allow.self.signed.certs" #define SYM_PARAMETER_SQLITE_BUSY_TIMEOUT_MS "sqlite.busy.timeout.ms" +#define SYM_PARAMETER_SQLITE_INIT_SQL "sqlite.init.sql" #endif diff --git a/symmetric-client-clib/inc/db/platform/sqlite/SqlitePlatform.h b/symmetric-client-clib/inc/db/platform/sqlite/SqlitePlatform.h index c71b004901..0114db21fd 100644 --- a/symmetric-client-clib/inc/db/platform/sqlite/SqlitePlatform.h +++ b/symmetric-client-clib/inc/db/platform/sqlite/SqlitePlatform.h @@ -27,6 +27,7 @@ #include "db/platform/DatabasePlatform.h" #include "db/platform/sqlite/SqliteDdlReader.h" #include "db/sqlite/SqliteSqlTemplate.h" +#include "util/StringArray.h" #define SYM_SQLITE_DEFAULT_BUSY_TIMEOUT_MS "30000" diff --git a/symmetric-client-clib/inc/model/Node.h b/symmetric-client-clib/inc/model/Node.h index c5d3a50d01..719e3ee4dc 100644 --- a/symmetric-client-clib/inc/model/Node.h +++ b/symmetric-client-clib/inc/model/Node.h @@ -23,7 +23,7 @@ #include -#define SYM_VERSION "3.7.27" +#define SYM_VERSION "3.7.27.1" typedef enum SymNodeStatus { SYM_NODE_STATUS_DATA_LOAD_NOT_STARTED, diff --git a/symmetric-client-clib/src/db/platform/sqlite/SqlitePlatform.c b/symmetric-client-clib/src/db/platform/sqlite/SqlitePlatform.c index 7bca0d2fa7..01a9cd2034 100644 --- a/symmetric-client-clib/src/db/platform/sqlite/SqlitePlatform.c +++ b/symmetric-client-clib/src/db/platform/sqlite/SqlitePlatform.c @@ -49,7 +49,7 @@ SymSqliteSqlTemplate * SymSqlitePlatform_getSqlTemplate(SymSqlitePlatform *this) void SymSqlitePlatform_destroy(SymDatabasePlatform *super) { SymLog_info("Closing SQLite database"); SymSqlitePlatform *this = (SymSqlitePlatform *) super; -// sqlite3_close(this->db); + sqlite3_close(this->db); this->sqlTemplate->destroy(this->sqlTemplate); // free(super->ddlReader); free(this); @@ -94,5 +94,18 @@ SymSqlitePlatform * SymSqlitePlatform_new(SymSqlitePlatform *this, SymProperties SymLog_info("Detected database '%s', version '%s'", super->name, super->version); + char *initSql = properties->get(properties, SYM_PARAMETER_SQLITE_INIT_SQL, NULL); + if (SymStringUtils_isNotBlank(initSql)) { + int error, i; + SymLog_debug("Initializing database with '%s'", initSql); + SymStringArray *array = SymStringArray_split(initSql, ";"); + for (i = 0; i < array->size; i++) { + char *sql = array->get(array, i); + if (SymStringUtils_isNotBlank(sql)) { + this->sqlTemplate->update(this->sqlTemplate, sql, NULL, NULL, &error); + } + } + } + return this; } From 8e53aad23a3da1c31502d0421e385596ef28c8ec Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Tue, 8 Dec 2015 13:42:09 -0500 Subject: [PATCH 19/51] 0002328: Error during use of dbexport feature on PostgreSQL database --- .../platform/postgresql/PostgresLobHandler.java | 16 ++++++++++------ .../org/jumpmind/db/sql/JdbcSqlTemplate.java | 6 +++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgresLobHandler.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgresLobHandler.java index e6a1a5ab61..4dc9920412 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgresLobHandler.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/postgresql/PostgresLobHandler.java @@ -40,14 +40,18 @@ public byte[] getBlobAsBytes(ResultSet rs, int columnIndex, int jdbcTypeCode, String jdbcTypeName) throws SQLException { if (PostgreSqlDatabasePlatform.isBlobStoredByReference(jdbcTypeName)) { - Blob blob = rs.getBlob(columnIndex); - if (blob != null) { - return blob.getBytes(1, (int) blob.length()); - } else { - return null; - } + return getLoColumnAsBytes(rs, columnIndex); } else { return getDefaultHandler().getBlobAsBytes(rs, columnIndex); } } + + public static byte[] getLoColumnAsBytes(ResultSet rs, int columnIndex) throws SQLException { + Blob blob = rs.getBlob(columnIndex); + if (blob != null) { + return blob.getBytes(1, (int) blob.length()); + } else { + return null; + } + } } diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java index 17c1041bc4..151e92a432 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/sql/JdbcSqlTemplate.java @@ -51,6 +51,7 @@ import org.apache.commons.io.IOUtils; import org.jumpmind.db.model.TypeMap; import org.jumpmind.db.platform.DatabaseInfo; +import org.jumpmind.db.platform.postgresql.PostgresLobHandler; import org.jumpmind.exception.IoException; import org.jumpmind.util.LinkedCaseInsensitiveMap; import org.slf4j.Logger; @@ -228,7 +229,6 @@ public String execute(Connection con) throws SQLException { public Map queryForMap(final String sql, final Object... args) { logSql(sql, args); return execute(new IConnectionCallback>() { - @SuppressWarnings("resource") public Map execute(Connection con) throws SQLException { Map result = null; PreparedStatement ps = null; @@ -247,6 +247,7 @@ public Map execute(Connection con) throws SQLException { for (int i = 1; i <= colCount; i++) { String key = meta.getColumnName(i); Object value = rs.getObject(i); + // TODO It seems like this should call getResultSetValue() if (value instanceof Blob) { Blob blob = (Blob) value; try { @@ -497,6 +498,7 @@ public static Object getResultSetValue(ResultSet rs, int index, boolean readStri ResultSetMetaData metaData = rs.getMetaData(); Object obj = null; int jdbcType = metaData.getColumnType(index); + String jdbcTypeName = metaData.getColumnTypeName(index); if (readStringsAsBytes && TypeMap.isTextType(jdbcType)) { byte[] bytes = rs.getBytes(index); if (bytes != null) { @@ -553,6 +555,8 @@ public static Object getResultSetValue(ResultSet rs, int index, boolean readStri if (typeName != null && typeName.equals("timestamptz")) { obj = rs.getString(index); } + } else if (jdbcTypeName != null && "oid".equals(jdbcTypeName)) { + obj = PostgresLobHandler.getLoColumnAsBytes(rs, index); } return obj; } From 923c7766206c9748e1b8b8e163880f1586460580 Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Tue, 8 Dec 2015 20:03:53 -0500 Subject: [PATCH 20/51] 0002464: If individual column templates are changed in a trigger template are changed from release to release, triggers are not rebuilt --- .../symmetric/db/AbstractTriggerTemplate.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java index 126e19be64..f97c4afca0 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java @@ -23,6 +23,7 @@ import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.commons.lang.StringUtils.isNotBlank; +import java.lang.reflect.Field; import java.sql.Types; import java.util.Map; @@ -1056,6 +1057,20 @@ public int toHashedValue() { for (String key : sqlTemplates.keySet()) { hashedValue += sqlTemplates.get(key).hashCode(); } + + Field[] fields = getClass().getSuperclass().getDeclaredFields(); + for (Field field : fields) { + if (field.getType().equals(String.class)) { + try { + String value = (String)field.get(this); + if (value != null) { + hashedValue += value.hashCode(); + } + } catch (Exception e) { + log.warn("Failed to get hash code for field " + field.getName()); + } + } + } } return hashedValue; } From e6bbf080f5ad20c2de285523a444eae5098548e4 Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Tue, 8 Dec 2015 20:05:11 -0500 Subject: [PATCH 21/51] 0002465: Add support for Oracle's XMLType --- .../jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java | 2 +- .../org/jumpmind/db/platform/oracle/OracleDmlStatement.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java index 6166fa88da..1bfc8d48d8 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java @@ -40,7 +40,7 @@ public OracleTriggerTemplate(ISymmetricDialect symmetricDialect) { dateTimeWithLocalTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; timeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; dateColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; - clobColumnTemplate = "decode(dbms_lob.getlength($(tableAlias).\"$(columnName)\"), null, to_clob(''), '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"')" ; + clobColumnTemplate = "decode(dbms_lob.getlength(to_clob($(tableAlias).\"$(columnName)\")), null, to_clob(''), '\"'||replace(replace($(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"')" ; blobColumnTemplate = "decode(dbms_lob.getlength($(tableAlias).\"$(columnName)\"), null, to_clob(''), '\"'||$(prefixName)_blob2clob($(tableAlias).\"$(columnName)\")||'\"')" ; booleanColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', '\"'||cast($(tableAlias).\"$(columnName)\" as number("+symmetricDialect.getTemplateNumberPrecisionSpec()+"))||'\"')" ; triggerConcatCharacter = "||" ; diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java index 20959412b4..893d3d69ec 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/oracle/OracleDmlStatement.java @@ -68,9 +68,11 @@ protected void appendColumnEquals(StringBuilder sql, Column column) { @Override protected int getTypeCode(Column column, boolean isDateOverrideToTimestamp) { int typeCode = super.getTypeCode(column, isDateOverrideToTimestamp); - if (typeCode == Types.LONGVARCHAR) { + if (column.getJdbcTypeName().startsWith("XML")) { + typeCode = Types.VARCHAR; + } else if (typeCode == Types.LONGVARCHAR) { typeCode = Types.CLOB; - } + } return typeCode; } From 9dbc7267f641f0338cd99f5a3d24f3463dc15cdd Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Wed, 9 Dec 2015 08:08:42 -0500 Subject: [PATCH 22/51] 0002466: Add property to disable setting the the h2.basedir system property --- .../java/org/jumpmind/symmetric/AbstractCommandLauncher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/AbstractCommandLauncher.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/AbstractCommandLauncher.java index de0bb74d6e..1a2d912258 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/AbstractCommandLauncher.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/AbstractCommandLauncher.java @@ -109,7 +109,7 @@ public abstract class AbstractCommandLauncher { symHome = "."; } System.setProperty("log4j.sym.home", symHome); - if (isBlank(System.getProperty("h2.baseDir"))) { + if (isBlank(System.getProperty("h2.baseDir.disable")) && isBlank(System.getProperty("h2.baseDir"))) { System.setProperty("h2.baseDir", symHome + "/tmp/h2"); } DEFAULT_SERVER_PROPERTIES = System.getProperty(SystemConstants.SYSPROP_SERVER_PROPERTIES_PATH, symHome + "/conf/symmetric-server.properties"); From c9a0ff54df9ba667df3cb661f50ba3f4f150014d Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Wed, 9 Dec 2015 21:31:09 -0500 Subject: [PATCH 23/51] 0002467: Issue 0002367 broke insert transformations --- .../org/jumpmind/symmetric/io/data/writer/TransformWriter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/TransformWriter.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/TransformWriter.java index 1d1743e739..f209a53372 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/TransformWriter.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/TransformWriter.java @@ -317,6 +317,8 @@ protected boolean perform(DataContext context, TransformedData data, TargetDmlAction targetAction = null; switch (data.getTargetDmlType()) { case INSERT: + targetAction = TargetDmlAction.INS_ROW; + break; case UPDATE: targetAction = transformation.evaluateTargetDmlAction(context, data); break; From d546e58d75b912cd86e705ce5fba3d719a219740 Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Wed, 9 Dec 2015 21:34:31 -0500 Subject: [PATCH 24/51] 0002453: Make the precision of the fractional millisecond capture on oracle timestamps configurable or default to FF (versus FF3) --- .../org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java index 1bfc8d48d8..4c1b4b066e 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/db/oracle/OracleTriggerTemplate.java @@ -35,7 +35,7 @@ public OracleTriggerTemplate(ISymmetricDialect symmetricDialect) { stringColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, $(oracleToClob)'', '\"'||replace(replace($(oracleToClob)$(tableAlias).\"$(columnName)\",'\\','\\\\'),'\"','\\\"')||'\"')" ; geometryColumnTemplate = "case when $(tableAlias).\"$(columnName)\" is null then to_clob('') else '\"'||replace(replace(SDO_UTIL.TO_WKTGEOMETRY($(tableAlias).\"$(columnName)\"),'\\','\\\\'),'\"','\\\"')||'\"' end"; numberColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', '\"'||cast($(tableAlias).\"$(columnName)\" as number("+symmetricDialect.getTemplateNumberPrecisionSpec()+"))||'\"')" ; - datetimeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF3')),'\"'))" ; + datetimeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; dateTimeWithTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM')),'\"'))" ; dateTimeWithLocalTimeZoneColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char(cast($(tableAlias).\"$(columnName)\" as timestamp), 'YYYY-MM-DD HH24:MI:SS.FF9')),'\"'))" ; timeColumnTemplate = "decode($(tableAlias).\"$(columnName)\", null, '', concat(concat('\"',to_char($(tableAlias).\"$(columnName)\", 'YYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=''GREGORIAN''')),'\"'))" ; From b7bcfbbf9f7638ab08fc84985aa32d168d433b74 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 10 Dec 2015 09:20:56 -0500 Subject: [PATCH 25/51] Fixes for OfflinePush and OfflinePull. --- symmetric-client-clib/inc/core/SymEngine.h | 2 ++ .../inc/service/NodeCommunicationService.h | 3 +- .../inc/service/OfflinePullService.h | 7 ++-- .../inc/service/OfflinePushService.h | 10 +++--- .../inc/service/PullService.h | 5 ++- .../inc/service/PushService.h | 4 ++- .../transport/file/FileIncomingTransport.h | 1 + .../transport/file/FileOutgoingTransport.h | 1 + symmetric-client-clib/inc/util/FileUtils.h | 32 +++++++++++++++++ symmetric-client-clib/src/core/SymEngine.c | 14 +++++--- .../src/service/NodeCommunicationService.c | 8 +++-- .../src/service/OfflinePullService.c | 8 +++-- .../src/service/OfflinePushService.c | 10 +++--- .../src/service/PullService.c | 6 ++-- .../src/service/PushService.c | 5 +-- .../transport/file/FileIncomingTransport.c | 15 ++++---- .../transport/file/FileOutgoingTransport.c | 31 ++++++++++++---- symmetric-client-clib/src/util/FileUtils.c | 36 +++++++++++++++++++ 18 files changed, 157 insertions(+), 41 deletions(-) create mode 100644 symmetric-client-clib/inc/util/FileUtils.h create mode 100644 symmetric-client-clib/src/util/FileUtils.c diff --git a/symmetric-client-clib/inc/core/SymEngine.h b/symmetric-client-clib/inc/core/SymEngine.h index 0ac69e6b1e..b4a9d850eb 100644 --- a/symmetric-client-clib/inc/core/SymEngine.h +++ b/symmetric-client-clib/inc/core/SymEngine.h @@ -35,6 +35,7 @@ #include "service/ParameterService.h" #include "service/PushService.h" #include "service/NodeService.h" +#include "service/NodeCommunicationService.h" #include "service/PullService.h" #include "service/RegistrationService.h" #include "service/RouterService.h" @@ -72,6 +73,7 @@ typedef struct SymEngine { SymOfflinePushService *offlinePushService; SymOfflinePullService *offlinePullService; SymNodeService *nodeService; + SymNodeCommunicationService *nodeCommunicationService; SymIncomingBatchService *incomingBatchService; SymOutgoingBatchService *outgoingBatchService; SymAcknowledgeService *acknowledgeService; diff --git a/symmetric-client-clib/inc/service/NodeCommunicationService.h b/symmetric-client-clib/inc/service/NodeCommunicationService.h index f52db8e0a9..ede4a2f4af 100644 --- a/symmetric-client-clib/inc/service/NodeCommunicationService.h +++ b/symmetric-client-clib/inc/service/NodeCommunicationService.h @@ -37,6 +37,7 @@ typedef struct SymNodeCommunicationService { void (*destroy)(struct SymNodeCommunicationService *this); } SymNodeCommunicationService; -SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, SymNodeService * nodeService); +SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, + SymNodeService * nodeService, SymParameterService *parameterService); #endif diff --git a/symmetric-client-clib/inc/service/OfflinePullService.h b/symmetric-client-clib/inc/service/OfflinePullService.h index 8f5759b7ef..53743aa4ab 100644 --- a/symmetric-client-clib/inc/service/OfflinePullService.h +++ b/symmetric-client-clib/inc/service/OfflinePullService.h @@ -29,6 +29,7 @@ #include "service/DataLoaderService.h" #include "service/RegistrationService.h" #include "service/ConfigurationService.h" +#include "service/NodeCommunicationService.h" #include "util/Map.h" #include "common/Log.h" @@ -37,12 +38,14 @@ typedef struct SymOfflinePullService { SymDataLoaderService *dataLoaderService; SymRegistrationService *registrationService; SymConfigurationService *configurationService; + SymNodeCommunicationService *nodeCommunicationService; SymRemoteNodeStatuses * (*pullData)(struct SymOfflinePullService *this); void (*destroy)(struct SymOfflinePullService *this); } SymOfflinePullService; -SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService); +SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymRegistrationService *registrationService, + SymConfigurationService *configurationService, SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/service/OfflinePushService.h b/symmetric-client-clib/inc/service/OfflinePushService.h index 249ea8a34b..2b176429b9 100644 --- a/symmetric-client-clib/inc/service/OfflinePushService.h +++ b/symmetric-client-clib/inc/service/OfflinePushService.h @@ -32,6 +32,7 @@ #include "service/ConfigurationService.h" #include "service/DataExtractorService.h" #include "service/AcknowledgeService.h" +#include "service/NodeCommunicationService.h" #include "transport/TransportManager.h" #include "transport/file/FileOutgoingTransport.h" #include "util/List.h" @@ -45,13 +46,14 @@ typedef struct SymOfflinePushService { SymParameterService *parameterService; SymConfigurationService *configurationService; SymAcknowledgeService *acknowledgeService; + SymNodeCommunicationService *nodeCommunicationService; SymRemoteNodeStatuses * (*pushData)(struct SymOfflinePushService *this); void (*destroy)(struct SymOfflinePushService *); } SymOfflinePushService; -SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, - SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService); - +SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, + SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, + SymParameterService *parameterService, SymConfigurationService *configurationService, + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/service/PullService.h b/symmetric-client-clib/inc/service/PullService.h index c5638fd41a..c7712fa5a1 100644 --- a/symmetric-client-clib/inc/service/PullService.h +++ b/symmetric-client-clib/inc/service/PullService.h @@ -29,6 +29,7 @@ #include "service/DataLoaderService.h" #include "service/RegistrationService.h" #include "service/ConfigurationService.h" +#include "service/NodeCommunicationService.h" #include "util/Map.h" #include "common/Log.h" @@ -37,11 +38,13 @@ typedef struct SymPullService { SymDataLoaderService *dataLoaderService; SymRegistrationService *registrationService; SymConfigurationService *configurationService; + SymNodeCommunicationService *nodeCommunicationService; SymRemoteNodeStatuses * (*pullData)(struct SymPullService *this); void (*destroy)(struct SymPullService *this); } SymPullService; SymPullService * SymPullService_new(SymPullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService); + SymRegistrationService *registrationService, SymConfigurationService *configurationService, + SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/service/PushService.h b/symmetric-client-clib/inc/service/PushService.h index 4069e206e7..fdb4610b18 100644 --- a/symmetric-client-clib/inc/service/PushService.h +++ b/symmetric-client-clib/inc/service/PushService.h @@ -27,6 +27,7 @@ #include "model/RemoteNodeStatus.h" #include "model/RemoteNodeStatuses.h" #include "service/NodeService.h" +#include "service/NodeCommunicationService.h" #include "service/ParameterService.h" #include "service/ConfigurationService.h" #include "service/DataExtractorService.h" @@ -37,6 +38,7 @@ #include "common/Log.h" typedef struct SymPushService { + SymNodeCommunicationService *nodeCommunicationService; SymNodeService *nodeService; SymDataExtractorService *dataExtractorService; SymTransportManager *transportManager; @@ -49,6 +51,6 @@ typedef struct SymPushService { SymPushService * SymPushService_new(SymPushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService); + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h index 5b33d8a0a9..dbacb9e786 100644 --- a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h +++ b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include diff --git a/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h b/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h index 0995a3c891..4a1a1441f7 100644 --- a/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h +++ b/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h @@ -22,6 +22,7 @@ #define SYM_FILEOUTGOINGTRANSPORT_H #include +#include #include "common/Log.h" #include "model/Node.h" #include "service/ParameterService.h" diff --git a/symmetric-client-clib/inc/util/FileUtils.h b/symmetric-client-clib/inc/util/FileUtils.h new file mode 100644 index 0000000000..974c2ba8ab --- /dev/null +++ b/symmetric-client-clib/inc/util/FileUtils.h @@ -0,0 +1,32 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU General Public License, version 3.0 (GPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU General Public License, + * version 3.0 (GPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef SYM_FILEUTILS_H +#define SYM_FILEUTILS_H + +#include +#include +#include +#include +#include "common/Log.h" + +int SymFileUtils_mkdir(char* dirName); + +#endif diff --git a/symmetric-client-clib/src/core/SymEngine.c b/symmetric-client-clib/src/core/SymEngine.c index 5489afd100..d86a414672 100644 --- a/symmetric-client-clib/src/core/SymEngine.c +++ b/symmetric-client-clib/src/core/SymEngine.c @@ -182,6 +182,7 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->offlineTransportManager = SymTransportManagerFactory_create(SYM_PROTOCOL_FILE, this->parameterService); this->nodeService = SymNodeService_new(NULL, this->platform); this->nodeService->lastRestartTime = SymDate_new(NULL); + this->nodeCommunicationService = SymNodeCommunicationService_new(NULL, this->nodeService, this->parameterService); this->incomingBatchService = SymIncomingBatchService_new(NULL, this->platform, this->parameterService); this->outgoingBatchService = SymOutgoingBatchService_new(NULL, this->platform, this->parameterService, this->sequenceService); this->acknowledgeService = SymAcknowledgeService_new(NULL, this->outgoingBatchService, this->platform); @@ -194,12 +195,15 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->triggerRouterService, this->parameterService, this->platform); this->registrationService = SymRegistrationService_new(NULL, this->nodeService, this->dataLoaderService, this->parameterService, this->configurationService); - this->pullService = SymPullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, this->configurationService); + this->pullService = SymPullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, + this->configurationService, this->nodeCommunicationService); this->pushService = SymPushService_new(NULL, this->nodeService, this->dataExtractorService, this->transportManager, this->parameterService, - this->configurationService, this->acknowledgeService); - this->offlinePullService = SymOfflinePullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, this->configurationService); - this->offlinePushService = SymOfflinePushService_new(NULL, this->nodeService, this->dataExtractorService, this->offlineTransportManager, this->parameterService, - this->configurationService, this->acknowledgeService); + this->configurationService, this->acknowledgeService, this->nodeCommunicationService); + this->offlinePullService = SymOfflinePullService_new(NULL, this->nodeService, + this->dataLoaderService, this->registrationService, this->configurationService, this->nodeCommunicationService); + this->offlinePushService = SymOfflinePushService_new(NULL, this->nodeService, + this->dataExtractorService, this->offlineTransportManager, this->parameterService, + this->configurationService, this->acknowledgeService, this->nodeCommunicationService); this->purgeService = SymPurgeService_new(NULL, this->parameterService, this->dialect, this->platform); return this; diff --git a/symmetric-client-clib/src/service/NodeCommunicationService.c b/symmetric-client-clib/src/service/NodeCommunicationService.c index 9155828a2b..7ccdec00a8 100644 --- a/symmetric-client-clib/src/service/NodeCommunicationService.c +++ b/symmetric-client-clib/src/service/NodeCommunicationService.c @@ -108,10 +108,10 @@ SymList * SymNodeCommunicationService_list(SymNodeCommunicationService *this, Sy nodesToCommunicateWith = SymNodeCommunicationService_removeOfflineNodes(this, this->nodeService->findNodesToPushTo(this->nodeService)); break; case SYM_COMMUNICATION_TYPE_OFFLN_PUSH: - nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_FILE_PUSH); + nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_PUSH); break; case SYM_COMMUNICATION_TYPE_OFFLN_PULL: - nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_FILE_PULL); + nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_PULL); break; default: nodesToCommunicateWith = SymList_new(NULL); @@ -125,11 +125,13 @@ void SymNodeCommunicationService_destroy(SymNodeCommunicationService *this) { free(this); } -SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, SymNodeService * nodeService) { +SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, + SymNodeService * nodeService, SymParameterService *parameterService) { if (this == NULL) { this = (SymNodeCommunicationService *) calloc(1, sizeof(SymNodeCommunicationService)); } this->nodeService = nodeService; + this->parameterService = parameterService; this->list = (void *) &SymNodeCommunicationService_list; this->destroy = (void *) &SymNodeCommunicationService_destroy; return this; diff --git a/symmetric-client-clib/src/service/OfflinePullService.c b/symmetric-client-clib/src/service/OfflinePullService.c index 419c2549f6..8146618ebb 100644 --- a/symmetric-client-clib/src/service/OfflinePullService.c +++ b/symmetric-client-clib/src/service/OfflinePullService.c @@ -48,7 +48,7 @@ SymRemoteNodeStatuses * SymOfflinePullService_pullData(SymOfflinePullService *th identity = this->nodeService->findIdentity(this->nodeService); } if (identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPull(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_OFFLN_PULL); SymMap *channels = this->configurationService->getChannels(this->configurationService, 0); statuses = SymRemoteNodeStatuses_new(NULL, channels); SymIterator *iter = nodes->iterator(nodes); @@ -68,8 +68,9 @@ void SymOfflinePullService_destroy(SymOfflinePullService *this) { free(this); } -SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService) { +SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymRegistrationService *registrationService, + SymConfigurationService *configurationService, SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymOfflinePullService *) calloc(1, sizeof(SymOfflinePullService)); } @@ -77,6 +78,7 @@ SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, S this->dataLoaderService = dataLoaderService; this->registrationService = registrationService; this->configurationService = configurationService; + this->nodeCommunicationService = nodeCommunicationService; this->pullData = (void *) &SymOfflinePullService_pullData; this->destroy = (void *) &SymOfflinePullService_destroy; return this; diff --git a/symmetric-client-clib/src/service/OfflinePushService.c b/symmetric-client-clib/src/service/OfflinePushService.c index be0c71aa37..06881dec39 100644 --- a/symmetric-client-clib/src/service/OfflinePushService.c +++ b/symmetric-client-clib/src/service/OfflinePushService.c @@ -107,7 +107,7 @@ SymRemoteNodeStatuses * SymOfflinePushService_pushData(SymOfflinePushService *th SymRemoteNodeStatuses *statuses = SymRemoteNodeStatuses_new(NULL, channels); SymNode *identity = this->nodeService->findIdentity(this->nodeService); if (identity && identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPushTo(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_OFFLN_PUSH); if (nodes->size > 0) { SymNodeSecurity *identitySecurity = this->nodeService->findNodeSecurity(this->nodeService, identity->nodeId); if (identitySecurity) { @@ -134,9 +134,10 @@ void SymOfflinePushService_destroy(SymOfflinePushService *this) { free(this); } -SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, - SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService) { +SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, + SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, + SymParameterService *parameterService, SymConfigurationService *configurationService, + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymOfflinePushService *) calloc(1, sizeof(SymOfflinePushService)); } @@ -146,6 +147,7 @@ SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, S this->parameterService = parameterService; this->configurationService = configurationService; this->acknowledgeService = acknowledgeService; + this->nodeCommunicationService = nodeCommunicationService; this->pushData = (void *) &SymOfflinePushService_pushData; this->destroy = (void *) &SymOfflinePushService_destroy; return this; diff --git a/symmetric-client-clib/src/service/PullService.c b/symmetric-client-clib/src/service/PullService.c index 0da1b42204..d52bc48b27 100644 --- a/symmetric-client-clib/src/service/PullService.c +++ b/symmetric-client-clib/src/service/PullService.c @@ -53,7 +53,7 @@ SymRemoteNodeStatuses * SymPullService_pullData(SymPullService *this) { identity = this->nodeService->findIdentity(this->nodeService); } if (identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPull(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_PULL); SymMap *channels = this->configurationService->getChannels(this->configurationService, 0); statuses = SymRemoteNodeStatuses_new(NULL, channels); SymIterator *iter = nodes->iterator(nodes); @@ -74,7 +74,8 @@ void SymPullService_destroy(SymPullService *this) { } SymPullService * SymPullService_new(SymPullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService) { + SymRegistrationService *registrationService, SymConfigurationService *configurationService, + SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymPullService *) calloc(1, sizeof(SymPullService)); } @@ -82,6 +83,7 @@ SymPullService * SymPullService_new(SymPullService *this, SymNodeService *nodeSe this->dataLoaderService = dataLoaderService; this->registrationService = registrationService; this->configurationService = configurationService; + this->nodeCommunicationService = nodeCommunicationService; this->pullData = (void *) &SymPullService_pullData; this->destroy = (void *) &SymPullService_destroy; return this; diff --git a/symmetric-client-clib/src/service/PushService.c b/symmetric-client-clib/src/service/PushService.c index 7ed52115e3..7d841b7b1a 100644 --- a/symmetric-client-clib/src/service/PushService.c +++ b/symmetric-client-clib/src/service/PushService.c @@ -107,7 +107,7 @@ SymRemoteNodeStatuses * SymPushService_pushData(SymPushService *this, unsigned i SymRemoteNodeStatuses *statuses = SymRemoteNodeStatuses_new(NULL, channels); SymNode *identity = this->nodeService->findIdentity(this->nodeService); if (identity && identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPushTo(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_PUSH); if (nodes->size > 0) { SymNodeSecurity *identitySecurity = this->nodeService->findNodeSecurity(this->nodeService, identity->nodeId); if (identitySecurity) { @@ -136,7 +136,7 @@ void SymPushService_destroy(SymPushService *this) { SymPushService * SymPushService_new(SymPushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService) { + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymPushService *) calloc(1, sizeof(SymPushService)); } @@ -146,6 +146,7 @@ SymPushService * SymPushService_new(SymPushService *this, SymNodeService *nodeSe this->parameterService = parameterService; this->configurationService = configurationService; this->acknowledgeService = acknowledgeService; + this->nodeCommunicationService = nodeCommunicationService; this->pushData = (void *) &SymPushService_pushData; this->destroy = (void *) &SymPushService_destroy; return this; diff --git a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c index 051bbab597..2e80e0f437 100644 --- a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c @@ -96,25 +96,28 @@ long SymFileIncomingTransport_process(SymFileIncomingTransport *this, SymDataPro processor->close(processor); fclose(file); + file = NULL; if (success) { if (SymStringUtils_isNotBlank(this->offlineArchiveDir)) { + SymFileUtils_mkdir(this->offlineArchiveDir); char *archivePath = SymStringUtils_format("%s/%s", this->offlineArchiveDir, fileName); int result = rename(path, archivePath); - if (result) { - SymLog_warn("Failed to archive '%s' to '%s'", path, archivePath); + if (result != 0) { + SymLog_warn("Failed to archive '%s' to '%s' %s", path, archivePath, strerror(errno)); } } else { int result = remove(path); - if (result) { - SymLog_warn("Failed to delete '%s'", path); + if (result != 0) { + SymLog_warn("Failed to delete '%s' %s", path, strerror(errno)); } } } else if (SymStringUtils_isNotBlank(this->offlineErrorDir)) { + SymFileUtils_mkdir(this->offlineErrorDir); char *errorPath = SymStringUtils_format("%s/%s", this->offlineErrorDir, fileName); int result = rename(path, errorPath); - if (result) { - SymLog_warn("Failed to archive '%s' to '%s'", path, errorPath); + if (result != 0) { + SymLog_warn("Failed to archive '%s' to '%s' %s", path, errorPath, strerror(errno)); } } diff --git a/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c b/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c index c1fe6be8be..9680ec48ba 100644 --- a/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c @@ -25,35 +25,52 @@ char *SymFileOutgoingTransport_getFileName(SymFileOutgoingTransport *this) { time(¤tTimeMillis); currentTimeMillis *= 1000; - return SymStringUtils_format("%s/%s-%s_to_%s-%s_%ld.tmp", this->offlineOutgoingDir, + return SymStringUtils_format("%s/%s-%s_to_%s-%s_%ld", this->offlineOutgoingDir, this->localNode->nodeGroupId, this->localNode->nodeId, this->remoteNode->nodeGroupId, this->remoteNode->nodeId, currentTimeMillis); } long SymFileOutgoingTransport_process(SymFileOutgoingTransport *this, SymDataProcessor *processor) { + SymFileUtils_mkdir(this->offlineOutgoingDir); + processor->open(processor); int BUFFER_SIZE = 2048; char inputBuffer[BUFFER_SIZE]; - char* fileName = SymFileOutgoingTransport_getFileName(this); - SymLog_debug("Writing file %s", fileName); + char* fileNameBase = SymFileOutgoingTransport_getFileName(this); + char *tmpFileName = SymStringUtils_format("%s%s", fileNameBase, ".tmp"); + char *csvFileName = SymStringUtils_format("%s%s", fileNameBase, ".csv"); + + SymLog_debug("Writing file %s", tmpFileName); long result = SYM_TRANSPORT_SC_SERVICE_UNAVAILABLE; - FILE *file = fopen(SymFileOutgoingTransport_getFileName(this), "w"); + FILE *file = fopen(tmpFileName, "w"); if (file) { int size; while ((size = processor->process(processor, inputBuffer, 1, BUFFER_SIZE)) > 0) { - fprintf(file, "%.*s", size, inputBuffer); + int result = fprintf(file, "%.*s", size, inputBuffer); + if (result < 0) { + SymLog_warn("failed to write to file %s rc=%d", tmpFileName, result); + } } fclose(file); result = SYM_TRANSPORT_OK; } else { - SymLog_error("Failed to open file for writing. %s", fileName); + SymLog_error("Failed to open file for writing. %s", fileNameBase); result = SYM_TRANSPORT_SC_SERVICE_UNAVAILABLE; } - free(fileName); + + SymLog_debug("Rename '%s' to '%s'", tmpFileName, csvFileName); + int renameResult = rename(tmpFileName, csvFileName); + if (renameResult != 0) { + SymLog_warn("Failed to rename '%s' to '%s' %s", tmpFileName, csvFileName, strerror(errno)); + } + + free(csvFileName); + free(tmpFileName); + free(fileNameBase); SymList *batchIds = processor->getBatchesProcessed(processor); SymStringBuilder *buff = SymStringBuilder_new(NULL); diff --git a/symmetric-client-clib/src/util/FileUtils.c b/symmetric-client-clib/src/util/FileUtils.c new file mode 100644 index 0000000000..6e3c9f1d3d --- /dev/null +++ b/symmetric-client-clib/src/util/FileUtils.c @@ -0,0 +1,36 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU General Public License, version 3.0 (GPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU General Public License, + * version 3.0 (GPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "util/FileUtils.h" + +int SymFileUtils_mkdir(char* dirName) { + + struct stat st = {0}; + int result = 0; + + if (stat(dirName, &st) == -1) { + result = mkdir(dirName, 0777); + if (result != 0) { + SymLog_warn("Failed to create dir '%s' %s", dirName, strerror(errno)); + } + } + + return result; +} From 8841095509c30c4352339989b692a54188790cce Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 10 Dec 2015 09:20:56 -0500 Subject: [PATCH 26/51] Fixes for OfflinePush and OfflinePull. --- symmetric-client-clib/inc/core/SymEngine.h | 2 ++ .../inc/service/NodeCommunicationService.h | 3 +- .../inc/service/OfflinePullService.h | 7 ++-- .../inc/service/OfflinePushService.h | 10 +++--- .../inc/service/PullService.h | 5 ++- .../inc/service/PushService.h | 4 ++- .../transport/file/FileIncomingTransport.h | 1 + .../transport/file/FileOutgoingTransport.h | 1 + symmetric-client-clib/inc/util/FileUtils.h | 32 +++++++++++++++++ symmetric-client-clib/src/core/SymEngine.c | 14 +++++--- .../src/service/NodeCommunicationService.c | 8 +++-- .../src/service/OfflinePullService.c | 8 +++-- .../src/service/OfflinePushService.c | 10 +++--- .../src/service/PullService.c | 6 ++-- .../src/service/PushService.c | 5 +-- .../transport/file/FileIncomingTransport.c | 15 ++++---- .../transport/file/FileOutgoingTransport.c | 31 ++++++++++++---- symmetric-client-clib/src/util/FileUtils.c | 36 +++++++++++++++++++ 18 files changed, 157 insertions(+), 41 deletions(-) create mode 100644 symmetric-client-clib/inc/util/FileUtils.h create mode 100644 symmetric-client-clib/src/util/FileUtils.c diff --git a/symmetric-client-clib/inc/core/SymEngine.h b/symmetric-client-clib/inc/core/SymEngine.h index 0ac69e6b1e..b4a9d850eb 100644 --- a/symmetric-client-clib/inc/core/SymEngine.h +++ b/symmetric-client-clib/inc/core/SymEngine.h @@ -35,6 +35,7 @@ #include "service/ParameterService.h" #include "service/PushService.h" #include "service/NodeService.h" +#include "service/NodeCommunicationService.h" #include "service/PullService.h" #include "service/RegistrationService.h" #include "service/RouterService.h" @@ -72,6 +73,7 @@ typedef struct SymEngine { SymOfflinePushService *offlinePushService; SymOfflinePullService *offlinePullService; SymNodeService *nodeService; + SymNodeCommunicationService *nodeCommunicationService; SymIncomingBatchService *incomingBatchService; SymOutgoingBatchService *outgoingBatchService; SymAcknowledgeService *acknowledgeService; diff --git a/symmetric-client-clib/inc/service/NodeCommunicationService.h b/symmetric-client-clib/inc/service/NodeCommunicationService.h index f52db8e0a9..ede4a2f4af 100644 --- a/symmetric-client-clib/inc/service/NodeCommunicationService.h +++ b/symmetric-client-clib/inc/service/NodeCommunicationService.h @@ -37,6 +37,7 @@ typedef struct SymNodeCommunicationService { void (*destroy)(struct SymNodeCommunicationService *this); } SymNodeCommunicationService; -SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, SymNodeService * nodeService); +SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, + SymNodeService * nodeService, SymParameterService *parameterService); #endif diff --git a/symmetric-client-clib/inc/service/OfflinePullService.h b/symmetric-client-clib/inc/service/OfflinePullService.h index 8f5759b7ef..53743aa4ab 100644 --- a/symmetric-client-clib/inc/service/OfflinePullService.h +++ b/symmetric-client-clib/inc/service/OfflinePullService.h @@ -29,6 +29,7 @@ #include "service/DataLoaderService.h" #include "service/RegistrationService.h" #include "service/ConfigurationService.h" +#include "service/NodeCommunicationService.h" #include "util/Map.h" #include "common/Log.h" @@ -37,12 +38,14 @@ typedef struct SymOfflinePullService { SymDataLoaderService *dataLoaderService; SymRegistrationService *registrationService; SymConfigurationService *configurationService; + SymNodeCommunicationService *nodeCommunicationService; SymRemoteNodeStatuses * (*pullData)(struct SymOfflinePullService *this); void (*destroy)(struct SymOfflinePullService *this); } SymOfflinePullService; -SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService); +SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymRegistrationService *registrationService, + SymConfigurationService *configurationService, SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/service/OfflinePushService.h b/symmetric-client-clib/inc/service/OfflinePushService.h index 249ea8a34b..2b176429b9 100644 --- a/symmetric-client-clib/inc/service/OfflinePushService.h +++ b/symmetric-client-clib/inc/service/OfflinePushService.h @@ -32,6 +32,7 @@ #include "service/ConfigurationService.h" #include "service/DataExtractorService.h" #include "service/AcknowledgeService.h" +#include "service/NodeCommunicationService.h" #include "transport/TransportManager.h" #include "transport/file/FileOutgoingTransport.h" #include "util/List.h" @@ -45,13 +46,14 @@ typedef struct SymOfflinePushService { SymParameterService *parameterService; SymConfigurationService *configurationService; SymAcknowledgeService *acknowledgeService; + SymNodeCommunicationService *nodeCommunicationService; SymRemoteNodeStatuses * (*pushData)(struct SymOfflinePushService *this); void (*destroy)(struct SymOfflinePushService *); } SymOfflinePushService; -SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, - SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService); - +SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, + SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, + SymParameterService *parameterService, SymConfigurationService *configurationService, + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/service/PullService.h b/symmetric-client-clib/inc/service/PullService.h index c5638fd41a..c7712fa5a1 100644 --- a/symmetric-client-clib/inc/service/PullService.h +++ b/symmetric-client-clib/inc/service/PullService.h @@ -29,6 +29,7 @@ #include "service/DataLoaderService.h" #include "service/RegistrationService.h" #include "service/ConfigurationService.h" +#include "service/NodeCommunicationService.h" #include "util/Map.h" #include "common/Log.h" @@ -37,11 +38,13 @@ typedef struct SymPullService { SymDataLoaderService *dataLoaderService; SymRegistrationService *registrationService; SymConfigurationService *configurationService; + SymNodeCommunicationService *nodeCommunicationService; SymRemoteNodeStatuses * (*pullData)(struct SymPullService *this); void (*destroy)(struct SymPullService *this); } SymPullService; SymPullService * SymPullService_new(SymPullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService); + SymRegistrationService *registrationService, SymConfigurationService *configurationService, + SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/service/PushService.h b/symmetric-client-clib/inc/service/PushService.h index 4069e206e7..fdb4610b18 100644 --- a/symmetric-client-clib/inc/service/PushService.h +++ b/symmetric-client-clib/inc/service/PushService.h @@ -27,6 +27,7 @@ #include "model/RemoteNodeStatus.h" #include "model/RemoteNodeStatuses.h" #include "service/NodeService.h" +#include "service/NodeCommunicationService.h" #include "service/ParameterService.h" #include "service/ConfigurationService.h" #include "service/DataExtractorService.h" @@ -37,6 +38,7 @@ #include "common/Log.h" typedef struct SymPushService { + SymNodeCommunicationService *nodeCommunicationService; SymNodeService *nodeService; SymDataExtractorService *dataExtractorService; SymTransportManager *transportManager; @@ -49,6 +51,6 @@ typedef struct SymPushService { SymPushService * SymPushService_new(SymPushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService); + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService); #endif diff --git a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h index 5b33d8a0a9..dbacb9e786 100644 --- a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h +++ b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include diff --git a/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h b/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h index 0995a3c891..4a1a1441f7 100644 --- a/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h +++ b/symmetric-client-clib/inc/transport/file/FileOutgoingTransport.h @@ -22,6 +22,7 @@ #define SYM_FILEOUTGOINGTRANSPORT_H #include +#include #include "common/Log.h" #include "model/Node.h" #include "service/ParameterService.h" diff --git a/symmetric-client-clib/inc/util/FileUtils.h b/symmetric-client-clib/inc/util/FileUtils.h new file mode 100644 index 0000000000..974c2ba8ab --- /dev/null +++ b/symmetric-client-clib/inc/util/FileUtils.h @@ -0,0 +1,32 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU General Public License, version 3.0 (GPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU General Public License, + * version 3.0 (GPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef SYM_FILEUTILS_H +#define SYM_FILEUTILS_H + +#include +#include +#include +#include +#include "common/Log.h" + +int SymFileUtils_mkdir(char* dirName); + +#endif diff --git a/symmetric-client-clib/src/core/SymEngine.c b/symmetric-client-clib/src/core/SymEngine.c index 5489afd100..d86a414672 100644 --- a/symmetric-client-clib/src/core/SymEngine.c +++ b/symmetric-client-clib/src/core/SymEngine.c @@ -182,6 +182,7 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->offlineTransportManager = SymTransportManagerFactory_create(SYM_PROTOCOL_FILE, this->parameterService); this->nodeService = SymNodeService_new(NULL, this->platform); this->nodeService->lastRestartTime = SymDate_new(NULL); + this->nodeCommunicationService = SymNodeCommunicationService_new(NULL, this->nodeService, this->parameterService); this->incomingBatchService = SymIncomingBatchService_new(NULL, this->platform, this->parameterService); this->outgoingBatchService = SymOutgoingBatchService_new(NULL, this->platform, this->parameterService, this->sequenceService); this->acknowledgeService = SymAcknowledgeService_new(NULL, this->outgoingBatchService, this->platform); @@ -194,12 +195,15 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->triggerRouterService, this->parameterService, this->platform); this->registrationService = SymRegistrationService_new(NULL, this->nodeService, this->dataLoaderService, this->parameterService, this->configurationService); - this->pullService = SymPullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, this->configurationService); + this->pullService = SymPullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, + this->configurationService, this->nodeCommunicationService); this->pushService = SymPushService_new(NULL, this->nodeService, this->dataExtractorService, this->transportManager, this->parameterService, - this->configurationService, this->acknowledgeService); - this->offlinePullService = SymOfflinePullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, this->configurationService); - this->offlinePushService = SymOfflinePushService_new(NULL, this->nodeService, this->dataExtractorService, this->offlineTransportManager, this->parameterService, - this->configurationService, this->acknowledgeService); + this->configurationService, this->acknowledgeService, this->nodeCommunicationService); + this->offlinePullService = SymOfflinePullService_new(NULL, this->nodeService, + this->dataLoaderService, this->registrationService, this->configurationService, this->nodeCommunicationService); + this->offlinePushService = SymOfflinePushService_new(NULL, this->nodeService, + this->dataExtractorService, this->offlineTransportManager, this->parameterService, + this->configurationService, this->acknowledgeService, this->nodeCommunicationService); this->purgeService = SymPurgeService_new(NULL, this->parameterService, this->dialect, this->platform); return this; diff --git a/symmetric-client-clib/src/service/NodeCommunicationService.c b/symmetric-client-clib/src/service/NodeCommunicationService.c index 9155828a2b..7ccdec00a8 100644 --- a/symmetric-client-clib/src/service/NodeCommunicationService.c +++ b/symmetric-client-clib/src/service/NodeCommunicationService.c @@ -108,10 +108,10 @@ SymList * SymNodeCommunicationService_list(SymNodeCommunicationService *this, Sy nodesToCommunicateWith = SymNodeCommunicationService_removeOfflineNodes(this, this->nodeService->findNodesToPushTo(this->nodeService)); break; case SYM_COMMUNICATION_TYPE_OFFLN_PUSH: - nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_FILE_PUSH); + nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_PUSH); break; case SYM_COMMUNICATION_TYPE_OFFLN_PULL: - nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_FILE_PULL); + nodesToCommunicateWith = SymNodeCommunicationService_getNodesToCommunicateWithOffline(this, SYM_COMMUNICATION_TYPE_PULL); break; default: nodesToCommunicateWith = SymList_new(NULL); @@ -125,11 +125,13 @@ void SymNodeCommunicationService_destroy(SymNodeCommunicationService *this) { free(this); } -SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, SymNodeService * nodeService) { +SymNodeCommunicationService * SymNodeCommunicationService_new(SymNodeCommunicationService *this, + SymNodeService * nodeService, SymParameterService *parameterService) { if (this == NULL) { this = (SymNodeCommunicationService *) calloc(1, sizeof(SymNodeCommunicationService)); } this->nodeService = nodeService; + this->parameterService = parameterService; this->list = (void *) &SymNodeCommunicationService_list; this->destroy = (void *) &SymNodeCommunicationService_destroy; return this; diff --git a/symmetric-client-clib/src/service/OfflinePullService.c b/symmetric-client-clib/src/service/OfflinePullService.c index 419c2549f6..8146618ebb 100644 --- a/symmetric-client-clib/src/service/OfflinePullService.c +++ b/symmetric-client-clib/src/service/OfflinePullService.c @@ -48,7 +48,7 @@ SymRemoteNodeStatuses * SymOfflinePullService_pullData(SymOfflinePullService *th identity = this->nodeService->findIdentity(this->nodeService); } if (identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPull(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_OFFLN_PULL); SymMap *channels = this->configurationService->getChannels(this->configurationService, 0); statuses = SymRemoteNodeStatuses_new(NULL, channels); SymIterator *iter = nodes->iterator(nodes); @@ -68,8 +68,9 @@ void SymOfflinePullService_destroy(SymOfflinePullService *this) { free(this); } -SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService) { +SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymRegistrationService *registrationService, + SymConfigurationService *configurationService, SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymOfflinePullService *) calloc(1, sizeof(SymOfflinePullService)); } @@ -77,6 +78,7 @@ SymOfflinePullService * SymOfflinePullService_new(SymOfflinePullService *this, S this->dataLoaderService = dataLoaderService; this->registrationService = registrationService; this->configurationService = configurationService; + this->nodeCommunicationService = nodeCommunicationService; this->pullData = (void *) &SymOfflinePullService_pullData; this->destroy = (void *) &SymOfflinePullService_destroy; return this; diff --git a/symmetric-client-clib/src/service/OfflinePushService.c b/symmetric-client-clib/src/service/OfflinePushService.c index be0c71aa37..06881dec39 100644 --- a/symmetric-client-clib/src/service/OfflinePushService.c +++ b/symmetric-client-clib/src/service/OfflinePushService.c @@ -107,7 +107,7 @@ SymRemoteNodeStatuses * SymOfflinePushService_pushData(SymOfflinePushService *th SymRemoteNodeStatuses *statuses = SymRemoteNodeStatuses_new(NULL, channels); SymNode *identity = this->nodeService->findIdentity(this->nodeService); if (identity && identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPushTo(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_OFFLN_PUSH); if (nodes->size > 0) { SymNodeSecurity *identitySecurity = this->nodeService->findNodeSecurity(this->nodeService, identity->nodeId); if (identitySecurity) { @@ -134,9 +134,10 @@ void SymOfflinePushService_destroy(SymOfflinePushService *this) { free(this); } -SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, - SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService) { +SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, SymNodeService *nodeService, + SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, + SymParameterService *parameterService, SymConfigurationService *configurationService, + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymOfflinePushService *) calloc(1, sizeof(SymOfflinePushService)); } @@ -146,6 +147,7 @@ SymOfflinePushService * SymOfflinePushService_new(SymOfflinePushService *this, S this->parameterService = parameterService; this->configurationService = configurationService; this->acknowledgeService = acknowledgeService; + this->nodeCommunicationService = nodeCommunicationService; this->pushData = (void *) &SymOfflinePushService_pushData; this->destroy = (void *) &SymOfflinePushService_destroy; return this; diff --git a/symmetric-client-clib/src/service/PullService.c b/symmetric-client-clib/src/service/PullService.c index 0da1b42204..d52bc48b27 100644 --- a/symmetric-client-clib/src/service/PullService.c +++ b/symmetric-client-clib/src/service/PullService.c @@ -53,7 +53,7 @@ SymRemoteNodeStatuses * SymPullService_pullData(SymPullService *this) { identity = this->nodeService->findIdentity(this->nodeService); } if (identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPull(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_PULL); SymMap *channels = this->configurationService->getChannels(this->configurationService, 0); statuses = SymRemoteNodeStatuses_new(NULL, channels); SymIterator *iter = nodes->iterator(nodes); @@ -74,7 +74,8 @@ void SymPullService_destroy(SymPullService *this) { } SymPullService * SymPullService_new(SymPullService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymRegistrationService *registrationService, SymConfigurationService *configurationService) { + SymRegistrationService *registrationService, SymConfigurationService *configurationService, + SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymPullService *) calloc(1, sizeof(SymPullService)); } @@ -82,6 +83,7 @@ SymPullService * SymPullService_new(SymPullService *this, SymNodeService *nodeSe this->dataLoaderService = dataLoaderService; this->registrationService = registrationService; this->configurationService = configurationService; + this->nodeCommunicationService = nodeCommunicationService; this->pullData = (void *) &SymPullService_pullData; this->destroy = (void *) &SymPullService_destroy; return this; diff --git a/symmetric-client-clib/src/service/PushService.c b/symmetric-client-clib/src/service/PushService.c index 7ed52115e3..7d841b7b1a 100644 --- a/symmetric-client-clib/src/service/PushService.c +++ b/symmetric-client-clib/src/service/PushService.c @@ -107,7 +107,7 @@ SymRemoteNodeStatuses * SymPushService_pushData(SymPushService *this, unsigned i SymRemoteNodeStatuses *statuses = SymRemoteNodeStatuses_new(NULL, channels); SymNode *identity = this->nodeService->findIdentity(this->nodeService); if (identity && identity->syncEnabled) { - SymList *nodes = this->nodeService->findNodesToPushTo(this->nodeService); + SymList *nodes = this->nodeCommunicationService->list(this->nodeCommunicationService, SYM_COMMUNICATION_TYPE_PUSH); if (nodes->size > 0) { SymNodeSecurity *identitySecurity = this->nodeService->findNodeSecurity(this->nodeService, identity->nodeId); if (identitySecurity) { @@ -136,7 +136,7 @@ void SymPushService_destroy(SymPushService *this) { SymPushService * SymPushService_new(SymPushService *this, SymNodeService *nodeService, SymDataExtractorService *dataExtractorService, SymTransportManager *transportManager, SymParameterService *parameterService, SymConfigurationService *configurationService, - SymAcknowledgeService *acknowledgeService) { + SymAcknowledgeService *acknowledgeService, SymNodeCommunicationService *nodeCommunicationService) { if (this == NULL) { this = (SymPushService *) calloc(1, sizeof(SymPushService)); } @@ -146,6 +146,7 @@ SymPushService * SymPushService_new(SymPushService *this, SymNodeService *nodeSe this->parameterService = parameterService; this->configurationService = configurationService; this->acknowledgeService = acknowledgeService; + this->nodeCommunicationService = nodeCommunicationService; this->pushData = (void *) &SymPushService_pushData; this->destroy = (void *) &SymPushService_destroy; return this; diff --git a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c index 051bbab597..2e80e0f437 100644 --- a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c @@ -96,25 +96,28 @@ long SymFileIncomingTransport_process(SymFileIncomingTransport *this, SymDataPro processor->close(processor); fclose(file); + file = NULL; if (success) { if (SymStringUtils_isNotBlank(this->offlineArchiveDir)) { + SymFileUtils_mkdir(this->offlineArchiveDir); char *archivePath = SymStringUtils_format("%s/%s", this->offlineArchiveDir, fileName); int result = rename(path, archivePath); - if (result) { - SymLog_warn("Failed to archive '%s' to '%s'", path, archivePath); + if (result != 0) { + SymLog_warn("Failed to archive '%s' to '%s' %s", path, archivePath, strerror(errno)); } } else { int result = remove(path); - if (result) { - SymLog_warn("Failed to delete '%s'", path); + if (result != 0) { + SymLog_warn("Failed to delete '%s' %s", path, strerror(errno)); } } } else if (SymStringUtils_isNotBlank(this->offlineErrorDir)) { + SymFileUtils_mkdir(this->offlineErrorDir); char *errorPath = SymStringUtils_format("%s/%s", this->offlineErrorDir, fileName); int result = rename(path, errorPath); - if (result) { - SymLog_warn("Failed to archive '%s' to '%s'", path, errorPath); + if (result != 0) { + SymLog_warn("Failed to archive '%s' to '%s' %s", path, errorPath, strerror(errno)); } } diff --git a/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c b/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c index c1fe6be8be..9680ec48ba 100644 --- a/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileOutgoingTransport.c @@ -25,35 +25,52 @@ char *SymFileOutgoingTransport_getFileName(SymFileOutgoingTransport *this) { time(¤tTimeMillis); currentTimeMillis *= 1000; - return SymStringUtils_format("%s/%s-%s_to_%s-%s_%ld.tmp", this->offlineOutgoingDir, + return SymStringUtils_format("%s/%s-%s_to_%s-%s_%ld", this->offlineOutgoingDir, this->localNode->nodeGroupId, this->localNode->nodeId, this->remoteNode->nodeGroupId, this->remoteNode->nodeId, currentTimeMillis); } long SymFileOutgoingTransport_process(SymFileOutgoingTransport *this, SymDataProcessor *processor) { + SymFileUtils_mkdir(this->offlineOutgoingDir); + processor->open(processor); int BUFFER_SIZE = 2048; char inputBuffer[BUFFER_SIZE]; - char* fileName = SymFileOutgoingTransport_getFileName(this); - SymLog_debug("Writing file %s", fileName); + char* fileNameBase = SymFileOutgoingTransport_getFileName(this); + char *tmpFileName = SymStringUtils_format("%s%s", fileNameBase, ".tmp"); + char *csvFileName = SymStringUtils_format("%s%s", fileNameBase, ".csv"); + + SymLog_debug("Writing file %s", tmpFileName); long result = SYM_TRANSPORT_SC_SERVICE_UNAVAILABLE; - FILE *file = fopen(SymFileOutgoingTransport_getFileName(this), "w"); + FILE *file = fopen(tmpFileName, "w"); if (file) { int size; while ((size = processor->process(processor, inputBuffer, 1, BUFFER_SIZE)) > 0) { - fprintf(file, "%.*s", size, inputBuffer); + int result = fprintf(file, "%.*s", size, inputBuffer); + if (result < 0) { + SymLog_warn("failed to write to file %s rc=%d", tmpFileName, result); + } } fclose(file); result = SYM_TRANSPORT_OK; } else { - SymLog_error("Failed to open file for writing. %s", fileName); + SymLog_error("Failed to open file for writing. %s", fileNameBase); result = SYM_TRANSPORT_SC_SERVICE_UNAVAILABLE; } - free(fileName); + + SymLog_debug("Rename '%s' to '%s'", tmpFileName, csvFileName); + int renameResult = rename(tmpFileName, csvFileName); + if (renameResult != 0) { + SymLog_warn("Failed to rename '%s' to '%s' %s", tmpFileName, csvFileName, strerror(errno)); + } + + free(csvFileName); + free(tmpFileName); + free(fileNameBase); SymList *batchIds = processor->getBatchesProcessed(processor); SymStringBuilder *buff = SymStringBuilder_new(NULL); diff --git a/symmetric-client-clib/src/util/FileUtils.c b/symmetric-client-clib/src/util/FileUtils.c new file mode 100644 index 0000000000..6e3c9f1d3d --- /dev/null +++ b/symmetric-client-clib/src/util/FileUtils.c @@ -0,0 +1,36 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU General Public License, version 3.0 (GPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU General Public License, + * version 3.0 (GPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "util/FileUtils.h" + +int SymFileUtils_mkdir(char* dirName) { + + struct stat st = {0}; + int result = 0; + + if (stat(dirName, &st) == -1) { + result = mkdir(dirName, 0777); + if (result != 0) { + SymLog_warn("Failed to create dir '%s' %s", dirName, strerror(errno)); + } + } + + return result; +} From 3777a8ef6e1c2a54f59839feb8c5eed3bc981c2a Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 10 Dec 2015 12:27:38 -0500 Subject: [PATCH 27/51] Fix offline path defaults and recursive directory creation. --- .../transport/file/FileIncomingTransport.h | 1 + .../transport/file/FileIncomingTransport.c | 3 ++ .../src/transport/file/FileTransportManager.c | 13 ++++----- symmetric-client-clib/src/util/FileUtils.c | 28 +++++++++++++++++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h index dbacb9e786..7318d7631c 100644 --- a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h +++ b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h @@ -33,6 +33,7 @@ #include "common/Log.h" #include "util/StringUtils.h" #include "util/StringArray.h" +#include "util/FileUtils.h" typedef struct SymFileIncomingTransport { SymIncomingTransport super; diff --git a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c index 2e80e0f437..1d1a4c9489 100644 --- a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c @@ -21,6 +21,9 @@ #include "transport/file/FileIncomingTransport.h" char* SymFileIncomingTransport_getIncomingFile(SymFileIncomingTransport *this, char *extension) { + + SymFileUtils_mkdir(this->offlineIncomingDir); + DIR *dir; struct dirent *dirEntries; dir = opendir(this->offlineIncomingDir); diff --git a/symmetric-client-clib/src/transport/file/FileTransportManager.c b/symmetric-client-clib/src/transport/file/FileTransportManager.c index 18dc142d16..22b2938069 100644 --- a/symmetric-client-clib/src/transport/file/FileTransportManager.c +++ b/symmetric-client-clib/src/transport/file/FileTransportManager.c @@ -24,23 +24,22 @@ int SymFileTransportManager_sendAcknowledgement(SymFileTransportManager *this, S return SYM_TRANSPORT_OK; } -char * SymFileTransportManager_getDirName(SymFileTransportManager *this, char *paramName, SymNode *localNode) { - // TODO replace nodeGroupId and nodeId variables. +char * SymFileTransportManager_getDirName(SymFileTransportManager *this, char *paramName, SymNode *localNode, char *defaultDirName) { SymTransportManager *super = &this->super; - char *dirName = super->parameterService->getString(super->parameterService, paramName, "."); + char *dirName = super->parameterService->getString(super->parameterService, paramName, defaultDirName); return dirName; } SymFileIncomingTransport * SymFileTransportManager_getPullTransport(SymFileTransportManager *this, SymNode *remote, SymNode *local, char *securityToken, SymProperties *requestProperties, char *registrationUrl) { return SymFileIncomingTransport_new(NULL, remote, local, - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_INCOMING_DIR, local), - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ARCHIVE_DIR, local), - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ERROR_DIR, local)); + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_INCOMING_DIR, local, "./tmp/offline/incoming"), + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ARCHIVE_DIR, local, ""), + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ERROR_DIR, local, "")); } SymFileOutgoingTransport * SymFileTransportManager_getPushTransport(SymFileTransportManager *this, SymNode *remote, SymNode *local, char *securityToken, char *registrationUrl) { return SymFileOutgoingTransport_new(NULL, remote, local, - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_OUTGOING_DIR, local)); + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_OUTGOING_DIR, local, "./tmp/offline/outgoing")); } SymList * SymFileTransportManager_readAcknowledgement(SymFileTransportManager *this, char *parameterString1, char *parameterString2) { diff --git a/symmetric-client-clib/src/util/FileUtils.c b/symmetric-client-clib/src/util/FileUtils.c index 6e3c9f1d3d..610e986948 100644 --- a/symmetric-client-clib/src/util/FileUtils.c +++ b/symmetric-client-clib/src/util/FileUtils.c @@ -22,13 +22,35 @@ int SymFileUtils_mkdir(char* dirName) { - struct stat st = {0}; int result = 0; + struct stat st = {0}; if (stat(dirName, &st) == -1) { - result = mkdir(dirName, 0777); + char tmp[256]; + char *p = NULL; + size_t len; + char SEPERATOR = '/'; + + snprintf(tmp, sizeof(tmp),"%s", dirName); + len = strlen(tmp); + if (tmp[len - 1] == SEPERATOR) { + tmp[len - 1] = 0; + } + + for (p = tmp + 1; *p; p++) { + if(*p == '/') { + *p = 0; + result = mkdir(tmp, S_IRWXU); + if (result != 0) { + SymLog_warn("Failed to create dir '%s' %s", tmp, strerror(errno)); + } + *p = '/'; + } + } + + result = mkdir(tmp, S_IRWXU); if (result != 0) { - SymLog_warn("Failed to create dir '%s' %s", dirName, strerror(errno)); + SymLog_warn("Failed to create dir '%s' %s", tmp, strerror(errno)); } } From 734d5847c1cb719b645fed8011c9ead5f679a549 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 10 Dec 2015 12:27:38 -0500 Subject: [PATCH 28/51] Fix offline path defaults and recursive directory creation. --- .../transport/file/FileIncomingTransport.h | 1 + .../transport/file/FileIncomingTransport.c | 3 ++ .../src/transport/file/FileTransportManager.c | 13 ++++----- symmetric-client-clib/src/util/FileUtils.c | 28 +++++++++++++++++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h index dbacb9e786..7318d7631c 100644 --- a/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h +++ b/symmetric-client-clib/inc/transport/file/FileIncomingTransport.h @@ -33,6 +33,7 @@ #include "common/Log.h" #include "util/StringUtils.h" #include "util/StringArray.h" +#include "util/FileUtils.h" typedef struct SymFileIncomingTransport { SymIncomingTransport super; diff --git a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c index 2e80e0f437..1d1a4c9489 100644 --- a/symmetric-client-clib/src/transport/file/FileIncomingTransport.c +++ b/symmetric-client-clib/src/transport/file/FileIncomingTransport.c @@ -21,6 +21,9 @@ #include "transport/file/FileIncomingTransport.h" char* SymFileIncomingTransport_getIncomingFile(SymFileIncomingTransport *this, char *extension) { + + SymFileUtils_mkdir(this->offlineIncomingDir); + DIR *dir; struct dirent *dirEntries; dir = opendir(this->offlineIncomingDir); diff --git a/symmetric-client-clib/src/transport/file/FileTransportManager.c b/symmetric-client-clib/src/transport/file/FileTransportManager.c index 18dc142d16..22b2938069 100644 --- a/symmetric-client-clib/src/transport/file/FileTransportManager.c +++ b/symmetric-client-clib/src/transport/file/FileTransportManager.c @@ -24,23 +24,22 @@ int SymFileTransportManager_sendAcknowledgement(SymFileTransportManager *this, S return SYM_TRANSPORT_OK; } -char * SymFileTransportManager_getDirName(SymFileTransportManager *this, char *paramName, SymNode *localNode) { - // TODO replace nodeGroupId and nodeId variables. +char * SymFileTransportManager_getDirName(SymFileTransportManager *this, char *paramName, SymNode *localNode, char *defaultDirName) { SymTransportManager *super = &this->super; - char *dirName = super->parameterService->getString(super->parameterService, paramName, "."); + char *dirName = super->parameterService->getString(super->parameterService, paramName, defaultDirName); return dirName; } SymFileIncomingTransport * SymFileTransportManager_getPullTransport(SymFileTransportManager *this, SymNode *remote, SymNode *local, char *securityToken, SymProperties *requestProperties, char *registrationUrl) { return SymFileIncomingTransport_new(NULL, remote, local, - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_INCOMING_DIR, local), - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ARCHIVE_DIR, local), - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ERROR_DIR, local)); + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_INCOMING_DIR, local, "./tmp/offline/incoming"), + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ARCHIVE_DIR, local, ""), + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_ERROR_DIR, local, "")); } SymFileOutgoingTransport * SymFileTransportManager_getPushTransport(SymFileTransportManager *this, SymNode *remote, SymNode *local, char *securityToken, char *registrationUrl) { return SymFileOutgoingTransport_new(NULL, remote, local, - SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_OUTGOING_DIR, local)); + SymFileTransportManager_getDirName(this, SYM_PARAMETER_NODE_OFFLINE_OUTGOING_DIR, local, "./tmp/offline/outgoing")); } SymList * SymFileTransportManager_readAcknowledgement(SymFileTransportManager *this, char *parameterString1, char *parameterString2) { diff --git a/symmetric-client-clib/src/util/FileUtils.c b/symmetric-client-clib/src/util/FileUtils.c index 6e3c9f1d3d..610e986948 100644 --- a/symmetric-client-clib/src/util/FileUtils.c +++ b/symmetric-client-clib/src/util/FileUtils.c @@ -22,13 +22,35 @@ int SymFileUtils_mkdir(char* dirName) { - struct stat st = {0}; int result = 0; + struct stat st = {0}; if (stat(dirName, &st) == -1) { - result = mkdir(dirName, 0777); + char tmp[256]; + char *p = NULL; + size_t len; + char SEPERATOR = '/'; + + snprintf(tmp, sizeof(tmp),"%s", dirName); + len = strlen(tmp); + if (tmp[len - 1] == SEPERATOR) { + tmp[len - 1] = 0; + } + + for (p = tmp + 1; *p; p++) { + if(*p == '/') { + *p = 0; + result = mkdir(tmp, S_IRWXU); + if (result != 0) { + SymLog_warn("Failed to create dir '%s' %s", tmp, strerror(errno)); + } + *p = '/'; + } + } + + result = mkdir(tmp, S_IRWXU); if (result != 0) { - SymLog_warn("Failed to create dir '%s' %s", dirName, strerror(errno)); + SymLog_warn("Failed to create dir '%s' %s", tmp, strerror(errno)); } } From 0c3ba3c194c6e9f737c1f56dac9b4c3f6533b36b Mon Sep 17 00:00:00 2001 From: elong Date: Fri, 11 Dec 2015 16:37:46 -0500 Subject: [PATCH 29/51] only heartbeat if registered --- .../src/service/DataService.c | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/symmetric-client-clib/src/service/DataService.c b/symmetric-client-clib/src/service/DataService.c index 14ccd7613a..255c6ee9eb 100644 --- a/symmetric-client-clib/src/service/DataService.c +++ b/symmetric-client-clib/src/service/DataService.c @@ -74,55 +74,55 @@ SymList * SymDataService_selectDataFor(SymDataService *this, SymBatch *batch) { void SymDataService_heartbeat(SymDataService *this, unsigned short force) { SymNode *me = this->nodeService->findIdentity(this->nodeService); - if (this->parameterService->is(this->parameterService, SYM_PARAMETER_HEARTBEAT_ENABLED, 1)) { - unsigned short updateWithBatchStatus = this->parameterService->is(this->parameterService, - SYM_PARAMETER_HEARTBEAT_UPDATE_NODE_WITH_BATCH_STATUS, 0); - - char *syncUrl = this->parameterService->getSyncUrl(this->parameterService); - char *schemaVersion = this->parameterService->getString(this->parameterService, SYM_PARAMETER_SCHEMA_VERSION, ""); - - int outgoingErrorCount = -1; - int outgoingUnsentCount = -1; - if (updateWithBatchStatus) { - outgoingUnsentCount = this->outgoingBatchService->countOutgoingBatchesUnsent(this->outgoingBatchService); - outgoingErrorCount = this->outgoingBatchService->countOutgoingBatchesInError(this->outgoingBatchService); - } + if (me != NULL) { + if (this->parameterService->is(this->parameterService, SYM_PARAMETER_HEARTBEAT_ENABLED, 1)) { + unsigned short updateWithBatchStatus = this->parameterService->is(this->parameterService, + SYM_PARAMETER_HEARTBEAT_UPDATE_NODE_WITH_BATCH_STATUS, 0); + + char *syncUrl = this->parameterService->getSyncUrl(this->parameterService); + char *schemaVersion = this->parameterService->getString(this->parameterService, SYM_PARAMETER_SCHEMA_VERSION, ""); + + int outgoingErrorCount = -1; + int outgoingUnsentCount = -1; + if (updateWithBatchStatus) { + outgoingUnsentCount = this->outgoingBatchService->countOutgoingBatchesUnsent(this->outgoingBatchService); + outgoingErrorCount = this->outgoingBatchService->countOutgoingBatchesInError(this->outgoingBatchService); + } - if (! SymStringUtils_equals(this->parameterService->getExternalId(this->parameterService), me->externalId) - || ! SymStringUtils_equals(this->parameterService->getNodeGroupId(this->parameterService), me->nodeGroupId) - || (syncUrl != NULL && ! SymStringUtils_equals(syncUrl, me->syncUrl)) - || ! SymStringUtils_equals(schemaVersion, me->schemaVersion) // TODO empty string vs. null is causing refresh everytime. - || ! SymStringUtils_equals(SYM_VERSION, me->symmetricVersion) - || ! SymStringUtils_equals(this->platform->name, me->databaseType) - || ! SymStringUtils_equals(this->platform->version, me->databaseVersion) - || me->batchInErrorCount != outgoingErrorCount - || me->batchToSendCount != outgoingUnsentCount) { - SymLog_info("Some attribute(s) of node changed. Recording changes"); - - me->deploymentType = SYM_DEPLOYMENT_TYPE; - me->symmetricVersion = SYM_VERSION; - me->databaseType = this->platform->name; - me->databaseVersion = this->platform->version; - me->batchInErrorCount = outgoingErrorCount; - me->batchToSendCount = outgoingUnsentCount; - me->schemaVersion = schemaVersion; - if (this->parameterService->is(this->parameterService, SYM_PARAMETER_AUTO_UPDATE_NODE_VALUES, 0)) { - SymLog_info("Updating my node configuration info according to the symmetric properties"); - me->externalId = this->parameterService->getExternalId(this->parameterService); - me->nodeGroupId = this->parameterService->getNodeGroupId(this->parameterService); - if (SymStringUtils_isNotBlank(this->parameterService->getSyncUrl(this->parameterService))) { - me->syncUrl = this->parameterService->getSyncUrl(this->parameterService); + if (! SymStringUtils_equals(this->parameterService->getExternalId(this->parameterService), me->externalId) + || ! SymStringUtils_equals(this->parameterService->getNodeGroupId(this->parameterService), me->nodeGroupId) + || (syncUrl != NULL && ! SymStringUtils_equals(syncUrl, me->syncUrl)) + || ! SymStringUtils_equals(schemaVersion, me->schemaVersion) // TODO empty string vs. null is causing refresh everytime. + || ! SymStringUtils_equals(SYM_VERSION, me->symmetricVersion) + || ! SymStringUtils_equals(this->platform->name, me->databaseType) + || ! SymStringUtils_equals(this->platform->version, me->databaseVersion) + || me->batchInErrorCount != outgoingErrorCount + || me->batchToSendCount != outgoingUnsentCount) { + SymLog_info("Some attribute(s) of node changed. Recording changes"); + + me->deploymentType = SYM_DEPLOYMENT_TYPE; + me->symmetricVersion = SYM_VERSION; + me->databaseType = this->platform->name; + me->databaseVersion = this->platform->version; + me->batchInErrorCount = outgoingErrorCount; + me->batchToSendCount = outgoingUnsentCount; + me->schemaVersion = schemaVersion; + if (this->parameterService->is(this->parameterService, SYM_PARAMETER_AUTO_UPDATE_NODE_VALUES, 0)) { + SymLog_info("Updating my node configuration info according to the symmetric properties"); + me->externalId = this->parameterService->getExternalId(this->parameterService); + me->nodeGroupId = this->parameterService->getNodeGroupId(this->parameterService); + if (SymStringUtils_isNotBlank(this->parameterService->getSyncUrl(this->parameterService))) { + me->syncUrl = this->parameterService->getSyncUrl(this->parameterService); + } } - } - this->nodeService->save(this->nodeService, me); + this->nodeService->save(this->nodeService, me); + } } - + SymLog_debug("Updating my node info"); + this->nodeService->updateNodeHostForCurrentNode(this->nodeService); + SymLog_debug("Done updating my node info"); } - - SymLog_debug("Updating my node info"); - this->nodeService->updateNodeHostForCurrentNode(this->nodeService); - SymLog_debug("Done updating my node info"); } void SymDataService_insertDataEvents(SymDataService *this, SymSqlTransaction *transaction, SymList *events) { From 25a02c4d5ed35011f62d7ef3f0eb301ccf86cefc Mon Sep 17 00:00:00 2001 From: elong Date: Fri, 11 Dec 2015 16:46:01 -0500 Subject: [PATCH 30/51] is sync_url is null then use registration_url --- symmetric-client-clib/src/transport/http/HttpTransportManager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/src/transport/http/HttpTransportManager.c b/symmetric-client-clib/src/transport/http/HttpTransportManager.c index 947e920149..d1d3941a82 100644 --- a/symmetric-client-clib/src/transport/http/HttpTransportManager.c +++ b/symmetric-client-clib/src/transport/http/HttpTransportManager.c @@ -34,7 +34,7 @@ static void append(SymStringBuilder *sb, char *name, char *value) { static char * buildUrl(char *action, SymNode *remote, SymNode *local, char *securityToken, char *registrationUrl) { SymStringBuilder *sb = SymStringBuilder_new(); - if (strcmp(remote->syncUrl, "") == 0) { + if (SymStringUtils_isBlank(remote->syncUrl)) { sb->append(sb, registrationUrl); } else { sb->append(sb, remote->syncUrl); From 96084eea059b1db96af1c937a651e085a5f2af05 Mon Sep 17 00:00:00 2001 From: elong Date: Fri, 11 Dec 2015 16:52:14 -0500 Subject: [PATCH 31/51] sync triggers after configuration change --- symmetric-client-clib/inc/io/writer/DataWriter.h | 1 + symmetric-client-clib/inc/model/Node.h | 2 +- .../inc/service/DataLoaderService.h | 4 +++- symmetric-client-clib/src/core/SymEngine.c | 4 ++-- .../src/io/writer/DefaultDatabaseWriter.c | 14 +++++++++++++- .../src/load/DefaultDataLoaderFactory.c | 3 ++- .../src/service/DataLoaderService.c | 7 ++++++- 7 files changed, 28 insertions(+), 7 deletions(-) diff --git a/symmetric-client-clib/inc/io/writer/DataWriter.h b/symmetric-client-clib/inc/io/writer/DataWriter.h index dd1718c8c3..c454abdf08 100644 --- a/symmetric-client-clib/inc/io/writer/DataWriter.h +++ b/symmetric-client-clib/inc/io/writer/DataWriter.h @@ -38,6 +38,7 @@ typedef struct SymDataWriter { void (*endTable)(struct SymDataWriter *this, SymTable *table); void (*endBatch)(struct SymDataWriter *this, SymBatch *batch); void (*destroy)(struct SymDataWriter *this); + unsigned short isSyncTriggersNeeded; } SymDataWriter; SymDataWriter * SymDataWriter_new(SymDataWriter *this); diff --git a/symmetric-client-clib/inc/model/Node.h b/symmetric-client-clib/inc/model/Node.h index 719e3ee4dc..a39f0c85c2 100644 --- a/symmetric-client-clib/inc/model/Node.h +++ b/symmetric-client-clib/inc/model/Node.h @@ -23,7 +23,7 @@ #include -#define SYM_VERSION "3.7.27.1" +#define SYM_VERSION "3.7.27.2" typedef enum SymNodeStatus { SYM_NODE_STATUS_DATA_LOAD_NOT_STARTED, diff --git a/symmetric-client-clib/inc/service/DataLoaderService.h b/symmetric-client-clib/inc/service/DataLoaderService.h index 646c2db4b1..2a9643f3da 100644 --- a/symmetric-client-clib/inc/service/DataLoaderService.h +++ b/symmetric-client-clib/inc/service/DataLoaderService.h @@ -27,6 +27,7 @@ #include "service/NodeService.h" #include "service/IncomingBatchService.h" #include "service/ParameterService.h" +#include "service/TriggerRouterService.h" #include "transport/TransportManager.h" #include "transport/IncomingTransport.h" #include "model/Node.h" @@ -44,6 +45,7 @@ typedef struct SymDataLoaderService { SymParameterService *parameterService; SymNodeService *nodeService; + SymTriggerRouterService *triggerRouterService; SymTransportManager *transportManager; SymTransportManager *fileTransportManager; SymDatabasePlatform *platform; @@ -56,7 +58,7 @@ typedef struct SymDataLoaderService { void (*destroy)(struct SymDataLoaderService *this); } SymDataLoaderService; -SymDataLoaderService * SymDataLoaderService_new(SymDataLoaderService *this, SymParameterService *parameterService, SymNodeService *nodeService, +SymDataLoaderService * SymDataLoaderService_new(SymDataLoaderService *this, SymParameterService *parameterService, SymNodeService *nodeService, SymTriggerRouterService *triggerRouterService, SymTransportManager *transportManager, SymTransportManager *fileTransportManager, SymDatabasePlatform *platform, SymDialect *dialect, SymIncomingBatchService *incomingBatchService); #endif diff --git a/symmetric-client-clib/src/core/SymEngine.c b/symmetric-client-clib/src/core/SymEngine.c index d86a414672..4ea408075c 100644 --- a/symmetric-client-clib/src/core/SymEngine.c +++ b/symmetric-client-clib/src/core/SymEngine.c @@ -186,8 +186,8 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->incomingBatchService = SymIncomingBatchService_new(NULL, this->platform, this->parameterService); this->outgoingBatchService = SymOutgoingBatchService_new(NULL, this->platform, this->parameterService, this->sequenceService); this->acknowledgeService = SymAcknowledgeService_new(NULL, this->outgoingBatchService, this->platform); - this->dataLoaderService = SymDataLoaderService_new(NULL, this->parameterService, this->nodeService, this->transportManager, this->offlineTransportManager, - this->platform, this->dialect, this->incomingBatchService); + this->dataLoaderService = SymDataLoaderService_new(NULL, this->parameterService, this->nodeService, this->triggerRouterService, + this->transportManager, this->offlineTransportManager, this->platform, this->dialect, this->incomingBatchService); this->dataService = SymDataService_new(NULL, this->platform, this->triggerRouterService, this->nodeService, this->dialect, this->outgoingBatchService, this->parameterService); this->routerService = SymRouterService_new(NULL, this->outgoingBatchService, this->sequenceService, this->dataService, this->nodeService, this->configurationService, this->parameterService, this->triggerRouterService, this->platform); diff --git a/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c b/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c index ee5e54d0bf..a6abb1a3b8 100644 --- a/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c +++ b/symmetric-client-clib/src/io/writer/DefaultDatabaseWriter.c @@ -207,6 +207,17 @@ unsigned short SymDefaultDatabaseWriter_write(SymDefaultDatabaseWriter *this, Sy } else { this->targetTable = this->sourceTable; } + } else if (!this->super.isSyncTriggersNeeded) { + unsigned short autoSync = this->parameterService->is(this->parameterService, SYM_PARAMETER_AUTO_SYNC_CONFIGURATION, 1) || + this->incomingBatch->batchId == SYM_VIRTUAL_BATCH_FOR_REGISTRATION; + if (autoSync && (SymStringUtils_equalsIgnoreCase(this->targetTable->name, SYM_TRIGGER) || + SymStringUtils_equalsIgnoreCase(this->targetTable->name, SYM_ROUTER) || + SymStringUtils_equalsIgnoreCase(this->targetTable->name, SYM_TRIGGER_ROUTER) || + SymStringUtils_equalsIgnoreCase(this->targetTable->name, SYM_TRIGGER_ROUTER_GROUPLET) || + SymStringUtils_equalsIgnoreCase(this->targetTable->name, SYM_GROUPLET_LINK) || + SymStringUtils_equalsIgnoreCase(this->targetTable->name, SYM_NODE_GROUP_LINK))) { + this->super.isSyncTriggersNeeded = 1; + } } int error = 0; switch (data->dataEventType) { @@ -292,12 +303,13 @@ void SymDefaultDatabaseWriter_destroy(SymDefaultDatabaseWriter *this) { } SymDefaultDatabaseWriter * SymDefaultDatabaseWriter_new(SymDefaultDatabaseWriter *this, SymIncomingBatchService *incomingBatchService, - SymDatabasePlatform *platform, SymDialect *dialect, SymDatabaseWriterSettings *settings) { + SymParameterService *parameterService, SymDatabasePlatform *platform, SymDialect *dialect, SymDatabaseWriterSettings *settings) { if (this == NULL) { this = (SymDefaultDatabaseWriter *) calloc(1, sizeof(SymDefaultDatabaseWriter)); } SymDataWriter *super = &this->super; this->incomingBatchService = incomingBatchService; + this->parameterService = parameterService; this->platform = platform; this->dialect = dialect; this->settings = settings; diff --git a/symmetric-client-clib/src/load/DefaultDataLoaderFactory.c b/symmetric-client-clib/src/load/DefaultDataLoaderFactory.c index e217d8ba0b..7aef3bbcd1 100644 --- a/symmetric-client-clib/src/load/DefaultDataLoaderFactory.c +++ b/symmetric-client-clib/src/load/DefaultDataLoaderFactory.c @@ -29,7 +29,8 @@ static SymDatabaseWriterSettings * buildDatabaseWriterSettings(SymDefaultDataLoa SymDataWriter * SymDefaultDataLoaderFactory_getDataWriter(SymDefaultDataLoaderFactory *this) { SymDatabaseWriterSettings *settings = buildDatabaseWriterSettings(this); - SymDataWriter *writer = (SymDataWriter *) SymDefaultDatabaseWriter_new(NULL, this->incomingBatchService, this->platform, this->dialect, settings); + SymDataWriter *writer = (SymDataWriter *) SymDefaultDatabaseWriter_new(NULL, this->incomingBatchService, this->parameterService, + this->platform, this->dialect, settings); return writer; } diff --git a/symmetric-client-clib/src/service/DataLoaderService.c b/symmetric-client-clib/src/service/DataLoaderService.c index a381fa894f..d1b55f927d 100644 --- a/symmetric-client-clib/src/service/DataLoaderService.c +++ b/symmetric-client-clib/src/service/DataLoaderService.c @@ -50,6 +50,10 @@ static SymList * SymDataLoaderService_loadDataFromTransport(SymDataLoaderService SymLog_debug("Transport rc = %ld" , rc); SymList *batchesProcessed = processor->getBatchesProcessed(processor); + if (writer->isSyncTriggersNeeded) { + this->triggerRouterService->syncTriggers(this->triggerRouterService, 0); + } + processor->destroy(processor); writer->destroy(writer); return batchesProcessed; @@ -150,13 +154,14 @@ void SymDataLoaderService_destroy(SymDataLoaderService *this) { free(this); } -SymDataLoaderService * SymDataLoaderService_new(SymDataLoaderService *this, SymParameterService *parameterService, SymNodeService *nodeService, +SymDataLoaderService * SymDataLoaderService_new(SymDataLoaderService *this, SymParameterService *parameterService, SymNodeService *nodeService, SymTriggerRouterService *triggerRouterService, SymTransportManager *transportManager, SymTransportManager *fileTransportManager, SymDatabasePlatform *platform, SymDialect *dialect, SymIncomingBatchService *incomingBatchService) { if (this == NULL) { this = (SymDataLoaderService *) calloc(1, sizeof(SymDataLoaderService)); } this->parameterService = parameterService; this->nodeService = nodeService; + this->triggerRouterService = triggerRouterService; this->transportManager = transportManager; this->fileTransportManager = fileTransportManager; this->platform = platform; From c171cf8c42c4d81ca143f3ee5552b12f8799ea4f Mon Sep 17 00:00:00 2001 From: elong Date: Fri, 11 Dec 2015 17:01:25 -0500 Subject: [PATCH 32/51] sync triggers after configuration change --- .../inc/io/writer/DefaultDatabaseWriter.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/io/writer/DefaultDatabaseWriter.h b/symmetric-client-clib/inc/io/writer/DefaultDatabaseWriter.h index 10faa481a9..dd170d3003 100644 --- a/symmetric-client-clib/inc/io/writer/DefaultDatabaseWriter.h +++ b/symmetric-client-clib/inc/io/writer/DefaultDatabaseWriter.h @@ -40,10 +40,15 @@ #include "db/SymDialect.h" #include "model/IncomingBatch.h" #include "service/IncomingBatchService.h" +#include "service/ParameterService.h" +#include "common/TableConstants.h" +#include "common/ParameterConstants.h" +#include "common/Constants.h" typedef struct SymDefaultDatabaseWriter { SymDataWriter super; SymIncomingBatchService *incomingBatchService; + SymParameterService *parameterService; SymDatabasePlatform *platform; SymDialect *dialect; SymDatabaseWriterSettings *settings; @@ -59,6 +64,6 @@ typedef struct SymDefaultDatabaseWriter { } SymDefaultDatabaseWriter; SymDefaultDatabaseWriter * SymDefaultDatabaseWriter_new(SymDefaultDatabaseWriter *this, SymIncomingBatchService *incomingBatchService, - SymDatabasePlatform *platform, SymDialect *dialect, SymDatabaseWriterSettings *settings); + SymParameterService *parameterService, SymDatabasePlatform *platform, SymDialect *dialect, SymDatabaseWriterSettings *settings); #endif From 70b90018c3fd1e812f09229c08bb6237726b091d Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Sat, 12 Dec 2015 15:00:27 -0500 Subject: [PATCH 33/51] 0002468: Set push.thread.per.server.count to 1. It should not have been bumped up to 100 in 2.7.22 --- symmetric-core/src/main/resources/symmetric-default.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties index c79f478102..9cbf864a67 100644 --- a/symmetric-core/src/main/resources/symmetric-default.properties +++ b/symmetric-core/src/main/resources/symmetric-default.properties @@ -807,7 +807,7 @@ pull.lock.timeout.ms=7200000 # # DatabaseOverridable: true # Tags: jobs -push.thread.per.server.count=100 +push.thread.per.server.count=1 # The amount of time a single push worker node_communication lock will timeout after. # From 9269eb73586ea9507b8a224d3d5b27b3def97b8b Mon Sep 17 00:00:00 2001 From: elong Date: Mon, 14 Dec 2015 14:02:55 -0500 Subject: [PATCH 34/51] add countIncomingBatchesInError --- symmetric-client-clib/inc/model/Node.h | 2 +- symmetric-client-clib/inc/service/IncomingBatchService.h | 3 +++ symmetric-client-clib/src/service/IncomingBatchService.c | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/model/Node.h b/symmetric-client-clib/inc/model/Node.h index a39f0c85c2..c33607b3aa 100644 --- a/symmetric-client-clib/inc/model/Node.h +++ b/symmetric-client-clib/inc/model/Node.h @@ -23,7 +23,7 @@ #include -#define SYM_VERSION "3.7.27.2" +#define SYM_VERSION "3.7.27.3" typedef enum SymNodeStatus { SYM_NODE_STATUS_DATA_LOAD_NOT_STARTED, diff --git a/symmetric-client-clib/inc/service/IncomingBatchService.h b/symmetric-client-clib/inc/service/IncomingBatchService.h index 83da9963af..98d149e788 100644 --- a/symmetric-client-clib/inc/service/IncomingBatchService.h +++ b/symmetric-client-clib/inc/service/IncomingBatchService.h @@ -41,6 +41,7 @@ typedef struct SymIncomingBatchService { int (*updateIncomingBatch)(struct SymIncomingBatchService *this, SymIncomingBatch *incomingBatch); int (*deleteIncomingBatch)(struct SymIncomingBatchService *this, SymIncomingBatch *incomingBatch); unsigned short (*isRecordOkBatchesEnabled)(struct SymIncomingBatchService *this); + int (*countIncomingBatchesInError)(struct SymIncomingBatchService *this); void (*destroy)(struct SymIncomingBatchService *this); } SymIncomingBatchService; @@ -69,4 +70,6 @@ where batch_id = ? and node_id = ?" #define SYM_SQL_DELETE_INCOMING_BATCH "delete from sym_incoming_batch where batch_id = ? and node_id = ?" +#define SYM_SQL_COUNT_INCOMING_BATCHES_ERRORS "select count(*) from sym_incoming_batch where error_flag = 1" + #endif diff --git a/symmetric-client-clib/src/service/IncomingBatchService.c b/symmetric-client-clib/src/service/IncomingBatchService.c index f941eb0302..977398c9ff 100644 --- a/symmetric-client-clib/src/service/IncomingBatchService.c +++ b/symmetric-client-clib/src/service/IncomingBatchService.c @@ -182,6 +182,12 @@ int SymIncomingBatchService_deleteIncomingBatch(SymIncomingBatchService *this, S return count; } +int SymIncomingBatchService_countIncomingBatchesInError(SymIncomingBatchService *this) { + SymSqlTemplate *sqlTemplate = this->platform->getSqlTemplate(this->platform); + int error; + return sqlTemplate->queryForInt(sqlTemplate, SYM_SQL_COUNT_INCOMING_BATCHES_ERRORS, NULL, NULL, &error); +} + void SymIncomingBatchService_destroy(SymIncomingBatchService *this) { free(this); } @@ -198,6 +204,7 @@ SymIncomingBatchService * SymIncomingBatchService_new(SymIncomingBatchService *t this->updateIncomingBatch = (void *) &SymIncomingBatchService_updateIncomingBatch; this->deleteIncomingBatch = (void *) &SymIncomingBatchService_deleteIncomingBatch; this->isRecordOkBatchesEnabled = (void *) &SymIncomingBatchService_isRecordOkBatchesEnabled; + this->countIncomingBatchesInError = (void *) &SymIncomingBatchService_countIncomingBatchesInError; this->destroy = (void *) &SymIncomingBatchService_destroy; return this; } From 14250107014929bd7e59de101f63c8b82cc35c43 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 15 Dec 2015 12:39:09 -0500 Subject: [PATCH 35/51] Perform initial heartbeat after registration. --- .../inc/service/RegistrationService.h | 7 +++++-- symmetric-client-clib/src/core/SymEngine.c | 2 +- .../src/service/RegistrationService.c | 10 ++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/symmetric-client-clib/inc/service/RegistrationService.h b/symmetric-client-clib/inc/service/RegistrationService.h index 0efcc929d8..f9bc282e5c 100644 --- a/symmetric-client-clib/inc/service/RegistrationService.h +++ b/symmetric-client-clib/inc/service/RegistrationService.h @@ -28,18 +28,21 @@ #include "service/DataLoaderService.h" #include "service/ConfigurationService.h" #include "model/Node.h" +#include "service/DataService.h" typedef struct SymRegistrationService { SymNodeService *nodeService; SymDataLoaderService *dataLoaderService; SymParameterService *parameterService; SymConfigurationService *configurationService; + SymDataService *dataService; void (*registerWithServer)(struct SymRegistrationService *this); unsigned short (*isRegisteredWithServer)(struct SymRegistrationService *this); void (*destroy)(struct SymRegistrationService *this); } SymRegistrationService; -SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymParameterService *parameterService, SymConfigurationService *configurationService); +SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymParameterService *parameterService, + SymConfigurationService *configurationService, SymDataService *dataService); #endif diff --git a/symmetric-client-clib/src/core/SymEngine.c b/symmetric-client-clib/src/core/SymEngine.c index 4ea408075c..5af570944a 100644 --- a/symmetric-client-clib/src/core/SymEngine.c +++ b/symmetric-client-clib/src/core/SymEngine.c @@ -194,7 +194,7 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->dataExtractorService = SymDataExtractorService_new(NULL, this->nodeService, this->outgoingBatchService, this->dataService, this->triggerRouterService, this->parameterService, this->platform); this->registrationService = SymRegistrationService_new(NULL, this->nodeService, this->dataLoaderService, this->parameterService, - this->configurationService); + this->configurationService, this->dataService); this->pullService = SymPullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, this->configurationService, this->nodeCommunicationService); this->pushService = SymPushService_new(NULL, this->nodeService, this->dataExtractorService, this->transportManager, this->parameterService, diff --git a/symmetric-client-clib/src/service/RegistrationService.c b/symmetric-client-clib/src/service/RegistrationService.c index 949ab94fac..7386d12e15 100644 --- a/symmetric-client-clib/src/service/RegistrationService.c +++ b/symmetric-client-clib/src/service/RegistrationService.c @@ -39,8 +39,8 @@ void SymRegistrationService_registerWithServer(SymRegistrationService *this) { if (isRegistered) { SymNode *node = this->nodeService->findIdentity(this->nodeService); if (node != NULL) { - SymLog_info("Successfully registered node [id=%s]\n", node->nodeId); - // TODO: this->dataService->heartbeat(this->dataService); + SymLog_info("Successfully registered node [id=%s]", node->nodeId); + this->dataService->heartbeat(this->dataService, 1); } else { SymLog_error("Node identity is missing after registration. The registration server may be misconfigured or have an error"); isRegistered = 0; @@ -73,8 +73,9 @@ void SymRegistrationService_destroy(SymRegistrationService *this) { free(this); } -SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymParameterService *parameterService, SymConfigurationService *configurationService) { +SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymParameterService *parameterService, + SymConfigurationService *configurationService, SymDataService *dataService) { if (this == NULL) { this = (SymRegistrationService *) calloc(1, sizeof(SymRegistrationService)); } @@ -82,6 +83,7 @@ SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this this->dataLoaderService = dataLoaderService; this->parameterService = parameterService; this->configurationService = configurationService; + this->dataService = dataService; this->isRegisteredWithServer = (void *) &SymRegistrationService_isRegisteredWithServer; this->registerWithServer = (void *) &SymRegistrationService_registerWithServer; this->destroy = (void *) &SymRegistrationService_destroy; From e9c3e45770d90c389a25f0704c065254520a2c13 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 15 Dec 2015 12:48:46 -0500 Subject: [PATCH 36/51] Fix to automatically detect z/OS DB2. --- .../org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java index 5147b8c85b..f60c6198d8 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java @@ -230,7 +230,9 @@ protected static String[] determineDatabaseNameVersionSubprotocol(DataSource dat } if (nameVersion[0].toLowerCase().indexOf(DatabaseNamesConstants.DB2) != -1) { - if (nameVersion[0].toUpperCase().indexOf("Z") != -1) { + String productVersion = getDatabaseProductVersion(dataSource); + if (nameVersion[0].toUpperCase().indexOf("Z") != -1 + || (productVersion != null && productVersion.startsWith("DSN"))) { nameVersion[0] = DatabaseNamesConstants.DB2ZOS; } else if (nameVersion[0].indexOf("400") != -1) { nameVersion[0] = DatabaseNamesConstants.DB2AS400; From f8d51cc15003cd043b7ac9224e16c0e9a0cd6f96 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 15 Dec 2015 12:39:09 -0500 Subject: [PATCH 37/51] Perform initial heartbeat after registration. --- .../inc/service/RegistrationService.h | 7 +++++-- symmetric-client-clib/src/core/SymEngine.c | 2 +- .../src/service/RegistrationService.c | 10 ++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/symmetric-client-clib/inc/service/RegistrationService.h b/symmetric-client-clib/inc/service/RegistrationService.h index 0efcc929d8..f9bc282e5c 100644 --- a/symmetric-client-clib/inc/service/RegistrationService.h +++ b/symmetric-client-clib/inc/service/RegistrationService.h @@ -28,18 +28,21 @@ #include "service/DataLoaderService.h" #include "service/ConfigurationService.h" #include "model/Node.h" +#include "service/DataService.h" typedef struct SymRegistrationService { SymNodeService *nodeService; SymDataLoaderService *dataLoaderService; SymParameterService *parameterService; SymConfigurationService *configurationService; + SymDataService *dataService; void (*registerWithServer)(struct SymRegistrationService *this); unsigned short (*isRegisteredWithServer)(struct SymRegistrationService *this); void (*destroy)(struct SymRegistrationService *this); } SymRegistrationService; -SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymParameterService *parameterService, SymConfigurationService *configurationService); +SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymParameterService *parameterService, + SymConfigurationService *configurationService, SymDataService *dataService); #endif diff --git a/symmetric-client-clib/src/core/SymEngine.c b/symmetric-client-clib/src/core/SymEngine.c index d86a414672..03cff36594 100644 --- a/symmetric-client-clib/src/core/SymEngine.c +++ b/symmetric-client-clib/src/core/SymEngine.c @@ -194,7 +194,7 @@ SymEngine * SymEngine_new( SymEngine *this, SymProperties *properties) { this->dataExtractorService = SymDataExtractorService_new(NULL, this->nodeService, this->outgoingBatchService, this->dataService, this->triggerRouterService, this->parameterService, this->platform); this->registrationService = SymRegistrationService_new(NULL, this->nodeService, this->dataLoaderService, this->parameterService, - this->configurationService); + this->configurationService, this->dataService); this->pullService = SymPullService_new(NULL, this->nodeService, this->dataLoaderService, this->registrationService, this->configurationService, this->nodeCommunicationService); this->pushService = SymPushService_new(NULL, this->nodeService, this->dataExtractorService, this->transportManager, this->parameterService, diff --git a/symmetric-client-clib/src/service/RegistrationService.c b/symmetric-client-clib/src/service/RegistrationService.c index 949ab94fac..7386d12e15 100644 --- a/symmetric-client-clib/src/service/RegistrationService.c +++ b/symmetric-client-clib/src/service/RegistrationService.c @@ -39,8 +39,8 @@ void SymRegistrationService_registerWithServer(SymRegistrationService *this) { if (isRegistered) { SymNode *node = this->nodeService->findIdentity(this->nodeService); if (node != NULL) { - SymLog_info("Successfully registered node [id=%s]\n", node->nodeId); - // TODO: this->dataService->heartbeat(this->dataService); + SymLog_info("Successfully registered node [id=%s]", node->nodeId); + this->dataService->heartbeat(this->dataService, 1); } else { SymLog_error("Node identity is missing after registration. The registration server may be misconfigured or have an error"); isRegistered = 0; @@ -73,8 +73,9 @@ void SymRegistrationService_destroy(SymRegistrationService *this) { free(this); } -SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, SymDataLoaderService *dataLoaderService, - SymParameterService *parameterService, SymConfigurationService *configurationService) { +SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this, SymNodeService *nodeService, + SymDataLoaderService *dataLoaderService, SymParameterService *parameterService, + SymConfigurationService *configurationService, SymDataService *dataService) { if (this == NULL) { this = (SymRegistrationService *) calloc(1, sizeof(SymRegistrationService)); } @@ -82,6 +83,7 @@ SymRegistrationService * SymRegistrationService_new(SymRegistrationService *this this->dataLoaderService = dataLoaderService; this->parameterService = parameterService; this->configurationService = configurationService; + this->dataService = dataService; this->isRegisteredWithServer = (void *) &SymRegistrationService_isRegisteredWithServer; this->registerWithServer = (void *) &SymRegistrationService_registerWithServer; this->destroy = (void *) &SymRegistrationService_destroy; From 650f5838f8b419b3963682323be11ffacf3fc1a7 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 15 Dec 2015 12:48:46 -0500 Subject: [PATCH 38/51] Fix to automatically detect z/OS DB2. --- .../org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java index 5147b8c85b..f60c6198d8 100644 --- a/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java +++ b/symmetric-jdbc/src/main/java/org/jumpmind/db/platform/JdbcDatabasePlatformFactory.java @@ -230,7 +230,9 @@ protected static String[] determineDatabaseNameVersionSubprotocol(DataSource dat } if (nameVersion[0].toLowerCase().indexOf(DatabaseNamesConstants.DB2) != -1) { - if (nameVersion[0].toUpperCase().indexOf("Z") != -1) { + String productVersion = getDatabaseProductVersion(dataSource); + if (nameVersion[0].toUpperCase().indexOf("Z") != -1 + || (productVersion != null && productVersion.startsWith("DSN"))) { nameVersion[0] = DatabaseNamesConstants.DB2ZOS; } else if (nameVersion[0].indexOf("400") != -1) { nameVersion[0] = DatabaseNamesConstants.DB2AS400; From 4c339c9fe8d64dbe88c6bc57d0bdd13f03e37776 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 17 Dec 2015 11:24:43 -0500 Subject: [PATCH 39/51] 0002469: routing.largest.gap.size is limited to an integer sized value, but should be a long. --- .../main/java/org/jumpmind/symmetric/route/DataGapDetector.java | 2 +- .../java/org/jumpmind/symmetric/service/impl/DataService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapDetector.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapDetector.java index 2080179aeb..36198c58d2 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapDetector.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapDetector.java @@ -93,7 +93,7 @@ public void beforeRouting() { final int dataIdIncrementBy = parameterService .getInt(ParameterConstants.DATA_ID_INCREMENT_BY); final long maxDataToSelect = parameterService - .getInt(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); + .getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); long databaseTime = symmetricDialect.getDatabaseTime(); int idsFilled = 0; int newGapsInserted = 0; diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java index 326cbf0844..c9448df036 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java @@ -1385,7 +1385,7 @@ public DataGap mapRow(Row rs) { public List findDataGaps() { final long maxDataToSelect = parameterService - .getInt(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); + .getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); List gaps = findDataGapsByStatus(DataGap.Status.GP); boolean lastGapExists = false; long maxDataEventId = 0; From 9c69bf5ba8d05c9781b28be3dd6362ba7ca09877 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 17 Dec 2015 17:40:04 -0500 Subject: [PATCH 40/51] Fix shifted table. --- .../src/asciidoc/configuration/load-filters/scripts.ad | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-assemble/src/asciidoc/configuration/load-filters/scripts.ad b/symmetric-assemble/src/asciidoc/configuration/load-filters/scripts.ad index b76b023826..699d90927e 100644 --- a/symmetric-assemble/src/asciidoc/configuration/load-filters/scripts.ad +++ b/symmetric-assemble/src/asciidoc/configuration/load-filters/scripts.ad @@ -27,7 +27,7 @@ Batch Rollback Script:: The script to execute if the batch rolls back. Handle Error Script:: A script to execute if data cannot be processed. .Variables available within scripts -[cols="3,^1,^1,5"] +[cols="3,^1,^1,^1,5"] |=== |Variable|BSH|SQL|JAVA|Description From 84cc40ce7ba81443b91ce03ca49025a1df96838c Mon Sep 17 00:00:00 2001 From: Chris Henson Date: Mon, 21 Dec 2015 16:19:25 -0500 Subject: [PATCH 41/51] grant access to bouser for weblogic --- .../src/main/java/org/jumpmind/db/model/Table.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/symmetric-db/src/main/java/org/jumpmind/db/model/Table.java b/symmetric-db/src/main/java/org/jumpmind/db/model/Table.java index f00b96d3a0..549e2f97fa 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/model/Table.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/model/Table.java @@ -32,12 +32,16 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Represents a table in the database model. */ public class Table implements Serializable, Cloneable, Comparable
{ + final static Logger log = LoggerFactory.getLogger(Table.class); + /** Unique ID for serialization purposes. */ private static final long serialVersionUID = -5541154961302342608L; @@ -1095,6 +1099,9 @@ public static Column[] orderColumns(String[] columnNames, Table table) { break; } } + if (orderedColumns[i] == null) { + log.warn("Could not find column with the name of {} on table {}", name, table.getName()); + } } return orderedColumns; } From 00baaa57f587af1194f385a24b3c6cb3dfdd7f48 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 22 Dec 2015 12:44:47 -0500 Subject: [PATCH 42/51] Support a rolling log file based on size. --- symmetric-client-clib/inc/common/Log.h | 8 +- symmetric-client-clib/inc/util/FileUtils.h | 2 + symmetric-client-clib/src/common/Log.c | 86 ++++++++++++++++++++++ symmetric-client-clib/src/util/FileUtils.c | 18 +++++ 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/common/Log.h b/symmetric-client-clib/inc/common/Log.h index 07eb003b6a..0f404a60b9 100644 --- a/symmetric-client-clib/inc/common/Log.h +++ b/symmetric-client-clib/inc/common/Log.h @@ -23,12 +23,15 @@ #define SYM_LOG_H #include -#include +#include #include +#include +#include #include "util/StringBuilder.h" #include "util/Properties.h" #include "util/Date.h" #include "util/StringUtils.h" +#include "util/FileUtils.h" typedef enum {SYM_LOG_LEVEL_DEBUG, SYM_LOG_LEVEL_INFO, SYM_LOG_LEVEL_WARN, SYM_LOG_LEVEL_ERROR} SymLogLevel; @@ -37,6 +40,9 @@ typedef enum {SYM_LOG_LEVEL_DEBUG, SYM_LOG_LEVEL_INFO, SYM_LOG_LEVEL_WARN, SYM_L #define SYM_LOG_SETTINGS_LOG_LEVEL "client.log.level" #define SYM_LOG_SETTINGS_LOG_DESTINATION "client.log.destination" #define SYM_LOG_SETTINGS_LOG_SHOW_SOURCE_FILE "client.log.show.source.file" +#define SYM_LOG_SETTINGS_LOG_MAX_FILE_SIZE "client.log.max.file.size" +#define SYM_LOG_SETTINGS_LOG_BACKUP_APPEND "client.log.backup.append" +#define SYM_LOG_SETTINGS_LOG_DAYS_TO_KEEP "client.log.backup.days.to.keep" #define SYM_LOG_LEVEL_DESC_DEBUG "DEBUG" #define SYM_LOG_LEVEL_DESC_INFO "INFO" diff --git a/symmetric-client-clib/inc/util/FileUtils.h b/symmetric-client-clib/inc/util/FileUtils.h index 974c2ba8ab..5b68ec505d 100644 --- a/symmetric-client-clib/inc/util/FileUtils.h +++ b/symmetric-client-clib/inc/util/FileUtils.h @@ -28,5 +28,7 @@ #include "common/Log.h" int SymFileUtils_mkdir(char* dirName); +int SymFileUtils_getFileSize(char *filename); +unsigned short SymFileUtils_exists(char *filename); #endif diff --git a/symmetric-client-clib/src/common/Log.c b/symmetric-client-clib/src/common/Log.c index e3d9c5b71a..97c7e60396 100644 --- a/symmetric-client-clib/src/common/Log.c +++ b/symmetric-client-clib/src/common/Log.c @@ -19,10 +19,14 @@ * under the License. */ #include "common/Log.h" +#include static int SymLog_logLevel = SYM_LOG_LEVEL_DEBUG; static unsigned short SymLog_showSourceFile = 0; static char* SymLog_destination = "console"; +static int SymLog_maxLogSize = 10485760; +static char* SymLog_backupAppend = "%Y-%m-%dT%H.%M.%S"; +static int SymLog_daysToKeep = 10; static char* SymLog_getlogLevelDescription(SymLogLevel logLevel) { switch (logLevel) { @@ -68,6 +72,84 @@ void SymLog_configure(SymProperties *settings) { SymLog_showSourceFile = SymStringUtils_equals(showSourceFile, "1") || SymStringUtils_equalsIgnoreCase(showSourceFile, "true"); } + + char *maxLogSize = settings->get(settings, SYM_LOG_SETTINGS_LOG_MAX_FILE_SIZE, "10485760"); + if (! SymStringUtils_isBlank(maxLogSize)) { + SymLog_maxLogSize = atoi(maxLogSize); + } + + char *backupAppend = settings->get(settings, SYM_LOG_SETTINGS_LOG_BACKUP_APPEND, "%Y-%m-%dT%H.%M.%S"); + if (! SymStringUtils_isBlank(backupAppend)) { + SymLog_backupAppend = backupAppend; + } + + char *daysToKeep = settings->get(settings, SYM_LOG_SETTINGS_LOG_DAYS_TO_KEEP, "10"); + if (! SymStringUtils_isBlank(daysToKeep)) { + SymLog_daysToKeep = atoi(daysToKeep); + } +} + +void SymLog_cleanupLogs(char *logFileName) { + if (SymLog_daysToKeep > 0) { + time_t nowInSeconds = time(NULL); + time_t timeToKeepInSeconds = SymLog_daysToKeep*24*60*60; + time_t logExpiryTime = nowInSeconds-timeToKeepInSeconds; + + char *logFileBaseName = basename(logFileName); + char *logDirName = dirname(logFileName); + DIR *logDir; + + logDir = opendir(logDirName); + struct dirent *directoryEntry; + + struct stat fileStatus; + + if (logDir != NULL) { + while ((directoryEntry = readdir(logDir)) != NULL) { + char *name = directoryEntry->d_name; + if (SymStringUtils_startsWith(name, logFileBaseName) + && ! SymStringUtils_equals(name, logFileBaseName)) { + + char *archivedLogFileName = SymStringUtils_format("%s/%s", logDirName, name); + stat(archivedLogFileName, &fileStatus); + if (fileStatus.st_mtime < logExpiryTime) { + int result = remove(archivedLogFileName); + if (result != 0) { + printf("Failed to remove '%s' %s\n", archivedLogFileName, strerror(errno)); + } + } + free(archivedLogFileName); + } + } + closedir(logDir); + } + } +} + +void SymLog_rollLogFile(char *logFileName) { + char *dateTimeString = calloc(24, sizeof(char)); + time_t now = time(NULL); + struct tm *timeInfo = localtime(&now); + strftime(dateTimeString, 24, SymLog_backupAppend, timeInfo); + + char *rolledFileName = SymStringUtils_format("%s%s", logFileName, dateTimeString); + + int result = rename(logFileName, rolledFileName); + if (result != 0) { + printf("Failed to rename '%s' to '%s' %s\n", logFileName, dateTimeString, + strerror(errno)); + } + + SymLog_cleanupLogs(logFileName); +} + +void SymLog_rollIfNeeded(char *logFileName) { + if (SymLog_maxLogSize > 0) { + int fileSize = SymFileUtils_getFileSize(logFileName); + if (fileSize > SymLog_maxLogSize) { + SymLog_rollLogFile(logFileName); + } + } } /** This is the central place where all logging funnels through. */ @@ -131,9 +213,13 @@ void SymLog_log(SymLogLevel logLevel, const char *functionName, const char *file fflush(destination); if (physicalFile) { fclose(destination); + SymLog_rollIfNeeded(SymLog_destination); + } date->destroy(date); messageBuffer->destroy(messageBuffer); } + + diff --git a/symmetric-client-clib/src/util/FileUtils.c b/symmetric-client-clib/src/util/FileUtils.c index 610e986948..6a1bc356c2 100644 --- a/symmetric-client-clib/src/util/FileUtils.c +++ b/symmetric-client-clib/src/util/FileUtils.c @@ -56,3 +56,21 @@ int SymFileUtils_mkdir(char* dirName) { return result; } + +int SymFileUtils_getFileSize(char *filename) { + struct stat st; + + if (stat(filename, &st) == 0) { + return st.st_size; + } + + return -1; +} + +unsigned short SymFileUtils_exists(char *filename) { + if( access( filename, F_OK ) != -1 ) { + return 1; + } else { + return 0; + } +} From 239bb6104578645d25ec629d0b65d2a082eb948b Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 22 Dec 2015 12:45:45 -0500 Subject: [PATCH 43/51] Fix include. --- symmetric-client-clib/inc/common/Log.h | 1 + symmetric-client-clib/src/common/Log.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/common/Log.h b/symmetric-client-clib/inc/common/Log.h index 0f404a60b9..21dd6a080b 100644 --- a/symmetric-client-clib/inc/common/Log.h +++ b/symmetric-client-clib/inc/common/Log.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "util/StringBuilder.h" #include "util/Properties.h" #include "util/Date.h" diff --git a/symmetric-client-clib/src/common/Log.c b/symmetric-client-clib/src/common/Log.c index 97c7e60396..ee22539d08 100644 --- a/symmetric-client-clib/src/common/Log.c +++ b/symmetric-client-clib/src/common/Log.c @@ -19,7 +19,6 @@ * under the License. */ #include "common/Log.h" -#include static int SymLog_logLevel = SYM_LOG_LEVEL_DEBUG; static unsigned short SymLog_showSourceFile = 0; From 66e4a6a53b6fb5dab3828b81d695e4e698ce95bc Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 22 Dec 2015 15:58:50 -0500 Subject: [PATCH 44/51] Fix bug where log file name might get corrupted after first roll. --- symmetric-client-clib/src/common/Log.c | 13 +++++++++---- symmetric-client-clib/src/service/PurgeService.c | 1 - 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/symmetric-client-clib/src/common/Log.c b/symmetric-client-clib/src/common/Log.c index ee22539d08..74d76fd5ee 100644 --- a/symmetric-client-clib/src/common/Log.c +++ b/symmetric-client-clib/src/common/Log.c @@ -94,8 +94,10 @@ void SymLog_cleanupLogs(char *logFileName) { time_t timeToKeepInSeconds = SymLog_daysToKeep*24*60*60; time_t logExpiryTime = nowInSeconds-timeToKeepInSeconds; - char *logFileBaseName = basename(logFileName); - char *logDirName = dirname(logFileName); + char* logFileNameClone1 = SymStringUtils_format("%s", logFileName); + char* logFileNameClone2 = SymStringUtils_format("%s", logFileName); + char *logFileBaseName = basename(logFileNameClone1); // basename() and dirname() may modify the input string. + char *logDirName = dirname(logFileNameClone2); DIR *logDir; logDir = opendir(logDirName); @@ -122,6 +124,8 @@ void SymLog_cleanupLogs(char *logFileName) { } closedir(logDir); } + free(logFileNameClone1); + free(logFileNameClone2); } } @@ -135,11 +139,12 @@ void SymLog_rollLogFile(char *logFileName) { int result = rename(logFileName, rolledFileName); if (result != 0) { - printf("Failed to rename '%s' to '%s' %s\n", logFileName, dateTimeString, + printf("Failed to rename '%s' to '%s' %s\n", logFileName, rolledFileName, strerror(errno)); } SymLog_cleanupLogs(logFileName); + free(rolledFileName); } void SymLog_rollIfNeeded(char *logFileName) { @@ -196,7 +201,7 @@ void SymLog_log(SymLogLevel logLevel, const char *functionName, const char *file else { destination = fopen(SymLog_destination, "a+"); if (!destination) { - printf("Failed to open log file destination '%s'. Check the path our use 'console'\n", SymLog_destination); + printf("Failed to open log file destination '%s'. Check the path or use 'console'\n", SymLog_destination); destination = stdout; } else { diff --git a/symmetric-client-clib/src/service/PurgeService.c b/symmetric-client-clib/src/service/PurgeService.c index 22052396ec..3c52120efb 100644 --- a/symmetric-client-clib/src/service/PurgeService.c +++ b/symmetric-client-clib/src/service/PurgeService.c @@ -307,7 +307,6 @@ SymDate *SymPurgeService_getRetentionCutoff(SymPurgeService *this) { long SymPurgeService_purgeIncoming(SymPurgeService *this) { long rowsPurged = 0; SymDate *retentionCutoff = SymPurgeService_getRetentionCutoff(this); - printf("%s\n", retentionCutoff->dateTimeString); rowsPurged += SymPurgeService_purgeIncomingBeforeDate(this, retentionCutoff); retentionCutoff->destroy(retentionCutoff); return rowsPurged; From c28f8261fe57c7557240fc827a7a847891a29645 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Tue, 22 Dec 2015 16:00:06 -0500 Subject: [PATCH 45/51] Formatting. --- symmetric-client-clib/src/common/Log.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/symmetric-client-clib/src/common/Log.c b/symmetric-client-clib/src/common/Log.c index 74d76fd5ee..24dfd8f8eb 100644 --- a/symmetric-client-clib/src/common/Log.c +++ b/symmetric-client-clib/src/common/Log.c @@ -94,8 +94,8 @@ void SymLog_cleanupLogs(char *logFileName) { time_t timeToKeepInSeconds = SymLog_daysToKeep*24*60*60; time_t logExpiryTime = nowInSeconds-timeToKeepInSeconds; - char* logFileNameClone1 = SymStringUtils_format("%s", logFileName); - char* logFileNameClone2 = SymStringUtils_format("%s", logFileName); + char *logFileNameClone1 = SymStringUtils_format("%s", logFileName); + char *logFileNameClone2 = SymStringUtils_format("%s", logFileName); char *logFileBaseName = basename(logFileNameClone1); // basename() and dirname() may modify the input string. char *logDirName = dirname(logFileNameClone2); DIR *logDir; From 86db2300e1a522b1c8435c08ebdb3470660d27b1 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 28 Dec 2015 14:05:38 -0500 Subject: [PATCH 46/51] Fix segfault when calling countOutgoingBatchesInError. --- symmetric-client-clib/src/service/OutgoingBatchService.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/symmetric-client-clib/src/service/OutgoingBatchService.c b/symmetric-client-clib/src/service/OutgoingBatchService.c index 0b6e13401d..8b6539dc7e 100644 --- a/symmetric-client-clib/src/service/OutgoingBatchService.c +++ b/symmetric-client-clib/src/service/OutgoingBatchService.c @@ -182,10 +182,13 @@ SymOutgoingBatchService * SymOutgoingBatchService_new(SymOutgoingBatchService *t this->platform = platform; this->parameterService = parameterService; this->sequenceService = sequenceService; + this->findOutgoingBatch = (void *) &SymOutgoingBatchService_findOutgoingBatch; this->getOutgoingBatches = (void *) &SymOutgoingBatchService_getOutgoingBatches; this->insertOutgoingBatch = (void *) &SymOutgoingBatchService_insertOutgoingBatch; this->updateOutgoingBatch = (void *) &SymOutgoingBatchService_updateOutgoingBatch; + this->countOutgoingBatchesUnsent = (void *) &SymOutgoingBatchService_countOutgoingBatchesUnsent; + this->countOutgoingBatchesInError = (void *) &SymOutgoingBatchService_countOutgoingBatchesInError; this->destroy = (void *) &SymOutgoingBatchService_destroy; return this; } From 52a334d0d41b4e12d12e6b072a4928781fbbf862 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 28 Dec 2015 14:06:34 -0500 Subject: [PATCH 47/51] Bump version. --- symmetric-client-clib/inc/model/Node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/model/Node.h b/symmetric-client-clib/inc/model/Node.h index c33607b3aa..1c1270cf8c 100644 --- a/symmetric-client-clib/inc/model/Node.h +++ b/symmetric-client-clib/inc/model/Node.h @@ -23,7 +23,7 @@ #include -#define SYM_VERSION "3.7.27.3" +#define SYM_VERSION "3.7.27.4" typedef enum SymNodeStatus { SYM_NODE_STATUS_DATA_LOAD_NOT_STARTED, From 594f434fe83f7e84f9b77a2feb16e0823ef6a821 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Mon, 28 Dec 2015 14:09:19 -0500 Subject: [PATCH 48/51] Cleanup. --- symmetric-client-clib/src/service/OutgoingBatchService.c | 1 - 1 file changed, 1 deletion(-) diff --git a/symmetric-client-clib/src/service/OutgoingBatchService.c b/symmetric-client-clib/src/service/OutgoingBatchService.c index 8b6539dc7e..eea5038ac8 100644 --- a/symmetric-client-clib/src/service/OutgoingBatchService.c +++ b/symmetric-client-clib/src/service/OutgoingBatchService.c @@ -182,7 +182,6 @@ SymOutgoingBatchService * SymOutgoingBatchService_new(SymOutgoingBatchService *t this->platform = platform; this->parameterService = parameterService; this->sequenceService = sequenceService; - this->findOutgoingBatch = (void *) &SymOutgoingBatchService_findOutgoingBatch; this->getOutgoingBatches = (void *) &SymOutgoingBatchService_getOutgoingBatches; this->insertOutgoingBatch = (void *) &SymOutgoingBatchService_insertOutgoingBatch; From 95485267de773aacc3799d666c4bfefd371d5995 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Wed, 30 Dec 2015 18:10:17 -0500 Subject: [PATCH 49/51] Fix for large BLOB handling and report proper encoding. --- symmetric-client-clib/inc/util/BinaryEncoding.h | 2 +- symmetric-client-clib/src/io/writer/ProtocolDataWriter.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/util/BinaryEncoding.h b/symmetric-client-clib/inc/util/BinaryEncoding.h index eab27aa92a..13c0bf9a24 100644 --- a/symmetric-client-clib/inc/util/BinaryEncoding.h +++ b/symmetric-client-clib/inc/util/BinaryEncoding.h @@ -27,7 +27,7 @@ #define SYM_BINARY_ENCODING_NONE "none" #define SYM_BINARY_ENCODING_BASE64 "base64" -#define SYM_BINARY_ENCODING_HEX "hex" +#define SYM_BINARY_ENCODING_HEX "HEX" typedef enum SymBinaryEncoding { SymBinaryEncoding_NONE, SymBinaryEncoding_BASE64, SymBinaryEncoding_HEX diff --git a/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c b/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c index 9aecf00792..82bc63252b 100644 --- a/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c +++ b/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c @@ -56,6 +56,7 @@ static void SymProtocolDataWriter_startBatch(SymProtocolDataWriter *this, SymBat if (this->isFirstBatch) { this->isFirstBatch = 0; SymProtocolDataWriter_println(this->sb, SYM_CSV_NODEID, this->sourceNodeId); + SymProtocolDataWriter_println(this->sb, SYM_CSV_BINARY, SYM_BINARY_ENCODING_HEX); } if (!SymStringUtils_isBlank(batch->channelId)) { SymProtocolDataWriter_println(this->sb, SYM_CSV_CHANNEL, batch->channelId); @@ -75,6 +76,8 @@ static void SymProtocolDataWriter_endBatch(SymProtocolDataWriter *this, SymBatch static void SymProtocolDataWriter_startTable(SymProtocolDataWriter *this, SymTable *table, SymBatch *batch) { if (!batch->isIgnore) { + this->table = table; + SymProtocolDataWriter_println(this->sb, SYM_CSV_CATALOG, table->catalog); SymProtocolDataWriter_println(this->sb, SYM_CSV_SCHEMA, table->schema); From 8f8460dd631d0c890b6cabc6a61e759525e5e631 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Wed, 30 Dec 2015 18:11:48 -0500 Subject: [PATCH 50/51] Bump version. # Conflicts: # symmetric-client-clib/inc/model/Node.h --- symmetric-client-clib/inc/model/Node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/inc/model/Node.h b/symmetric-client-clib/inc/model/Node.h index 1c1270cf8c..e4084a1ffd 100644 --- a/symmetric-client-clib/inc/model/Node.h +++ b/symmetric-client-clib/inc/model/Node.h @@ -23,7 +23,7 @@ #include -#define SYM_VERSION "3.7.27.4" +#define SYM_VERSION "3.7.27.5" typedef enum SymNodeStatus { SYM_NODE_STATUS_DATA_LOAD_NOT_STARTED, From 73993256125dbbdda2238192c380fa79e6bd3079 Mon Sep 17 00:00:00 2001 From: mmichalek Date: Thu, 31 Dec 2015 09:58:14 -0500 Subject: [PATCH 51/51] Correct debug statement to only log the current chunk. --- symmetric-client-clib/src/io/writer/ProtocolDataWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c b/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c index 82bc63252b..97556f14b8 100644 --- a/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c +++ b/symmetric-client-clib/src/io/writer/ProtocolDataWriter.c @@ -160,7 +160,7 @@ size_t SymProtocolDataWriter_process(SymProtocolDataWriter *this, char *buffer, if (numBytes > 0) { memcpy(buffer, this->sb->str, numBytes); - SymLog_debug("Writing data: %s\n", this->sb->str); + SymLog_debug("Writing data: %.*s\n", numBytes, this->sb->str); } if (bufferFull) {