0000776: Change SqlScriptReader to allow comments in places other tha…

…n the beginning of a line in a sql script

0000775: Change SqlScript to use SqlScriptReader so that statements are streamed

symmetric-client/src/main/java/org/jumpmind/symmetric/DbFill.java

@@ -1,7 +1,6 @@
 package org.jumpmind.symmetric;
 
 import java.math.BigDecimal;
-import java.math.BigInteger;
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.Date;
@@ -94,14 +93,22 @@ private void fillTables(Table... tables) {
         for (int i = 0; i < inputLength; i++) {
 
             for (Table table : tables) {
+                DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, table);
+
                 Column[] tableColumns = table.getColumns();
                 Object[] columnValues = generateRandomValues(insertedColumns, table);
                 for (int j = 0; j < tableColumns.length; j++) {
-                    insertedColumns.put(table.getName() + "." + tableColumns[j].getName(),
+                    insertedColumns.put(table.getQualifiedColumnName(tableColumns[j]),
                             columnValues[j]);
                 }
-                DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, table);
-                sqlTemplate.update(statement.getSql(), columnValues);
+
+                Column[] statementColumns = statement.getMetaData();
+                Object[] statementValues = new Object[statementColumns.length];
+                for (int j = 0; j < statementColumns.length; j++) {
+                    statementValues[j] = insertedColumns.get(table
+                            .getQualifiedColumnName(statementColumns[j]));
+                }
+                sqlTemplate.update(statement.getSql(), statementValues);
             }
 
             insertedColumns.clear();

symmetric-core/src/main/java/org/jumpmind/symmetric/AbstractSymmetricEngine.java

@@ -18,6 +18,7 @@
 import org.jumpmind.db.platform.IDatabasePlatform;
 import org.jumpmind.db.sql.ISqlTemplate;
 import org.jumpmind.db.sql.SqlScript;
+import org.jumpmind.db.sql.SqlScriptReader;
 import org.jumpmind.properties.TypedProperties;
 import org.jumpmind.symmetric.common.Constants;
 import org.jumpmind.symmetric.common.ParameterConstants;
@@ -441,7 +442,7 @@ protected boolean loadFromScriptIfProvided() {
 
                     if (fileUrl != null) {
                         new SqlScript(fileUrl, symmetricDialect.getPlatform().getSqlTemplate(),
-                                true, SqlScript.QUERY_ENDS, getSymmetricDialect().getPlatform()
+                                true, SqlScriptReader.QUERY_ENDS, getSymmetricDialect().getPlatform()
                                         .getSqlScriptReplacementTokens()).execute();
                         loaded = true;
                     } else {

symmetric-core/src/test/resources/test-data-drop-all.sql

@@ -1,32 +1,32 @@
---drop table sym_transform_column;
---drop table sym_transform_table;
---drop table sym_data_gap;
---drop table sym_node_channel_ctl;
---drop table sym_node_group_channel_window;
---drop table sym_data_event;
---drop table sym_trigger_hist;
---drop table sym_trigger_router;
---drop table sym_trigger;
---drop table sym_router;
---drop table sym_node_security;
---drop table sym_node_identity;
---drop table sym_lock;
---drop table sym_node_communication;
---drop table sym_node_host;
---drop table sym_node;
---drop table sym_conflict;
---drop table sym_node_group_link;
---drop table sym_node_group;
---drop table sym_incoming_batch;
---drop table sym_channel;
---drop table sym_outgoing_batch;
---drop table sym_parameter;
---drop table sym_node_host_channel_stats;
---drop table sym_node_host_stats;
---drop table sym_node_host_job_stats;
---drop table sym_registration_redirect;
---drop table sym_registration_request;
---drop table sym_data;
---drop table sym_incoming_error;
---drop table sym_sequence;
---drop table sym_load_filter;
+drop table sym_transform_column;
+drop table sym_transform_table;
+drop table sym_data_gap;
+drop table sym_node_channel_ctl;
+drop table sym_node_group_channel_window;
+drop table sym_data_event;
+drop table sym_trigger_hist;
+drop table sym_trigger_router;
+drop table sym_trigger;
+drop table sym_router;
+drop table sym_node_security;
+drop table sym_node_identity;
+drop table sym_lock;
+drop table sym_node_communication;
+drop table sym_node_host;
+drop table sym_node;
+drop table sym_conflict;
+drop table sym_node_group_link;
+drop table sym_node_group;
+drop table sym_incoming_batch;
+drop table sym_channel;
+drop table sym_outgoing_batch;
+drop table sym_parameter;
+drop table sym_node_host_channel_stats;
+drop table sym_node_host_stats;
+drop table sym_node_host_job_stats;
+drop table sym_registration_redirect;
+drop table sym_registration_request;
+drop table sym_data;
+drop table sym_incoming_error;
+drop table sym_sequence;
+drop table sym_load_filter;

symmetric-db/src/main/java/org/jumpmind/db/sql/ISqlStatementSource.java

@@ -0,0 +1,6 @@
+package org.jumpmind.db.sql;
+
+public interface ISqlStatementSource {
+
+    public String readSqlStatement();
+}

symmetric-db/src/main/java/org/jumpmind/db/sql/ISqlTemplate.java

@@ -61,6 +61,8 @@ public <T, W> Map<T, W> query(String sql, String keyCol, String valueCol, Object
 
     public int update(boolean autoCommit, boolean failOnError, int commitRate, ISqlResultsListener listener, String... sql);
 
+    public int update(boolean autoCommit, boolean failOnError, int commitRate, ISqlResultsListener listener, ISqlStatementSource source);
+
     public int update(boolean autoCommit, boolean failOnError, int commitRate, String... sql);
 
     public int update(String sql, Object[] values, int[] types);

symmetric-db/src/main/java/org/jumpmind/db/sql/ListSqlStatementSource.java

@@ -0,0 +1,25 @@
+package org.jumpmind.db.sql;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ListSqlStatementSource implements ISqlStatementSource {
+
+    protected List<String> statements;
+
+    public ListSqlStatementSource(String... statements) {
+        this.statements = new ArrayList<String>();
+        for (String sql : statements) {
+            this.statements.add(sql);
+        }
+    }
+
+    public ListSqlStatementSource(List<String> statements) {
+        this.statements = new ArrayList<String>(statements);
+    }
+
+    public String readSqlStatement() {
+        return statements.size() > 0 ? statements.remove(0) : null;
+    }
+
+}

symmetric-db/src/main/java/org/jumpmind/db/sql/SqlScript.java

@@ -22,47 +22,35 @@
 
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.Reader;
 import java.io.StringReader;
 import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.jumpmind.util.FormatUtils;
 
 /**
  * This class parses and runs SQL from an input file or buffer using the
  * designed {@link ISqlTemplate}.
  */
 public class SqlScript {
 
-    static final String COMMENT_CHARS_1 = "--";
-    static final String COMMENT_CHARS_2 = "#";
-
-    public final static String QUERY_ENDS = ";";
-
-    private String delimiter = QUERY_ENDS;
-
-    private List<String> statements;
-
     private ISqlTemplate sqlTemplate;
 
     private int commitRate = 10000;
 
     private boolean failOnError = true;
 
-    private String lineDeliminator;
-
     private ISqlResultsListener resultsListener;
 
+    private SqlScriptReader scriptReader;
+
     public SqlScript(URL url, ISqlTemplate sqlTemplate) {
-        this(url, sqlTemplate, true, QUERY_ENDS, null);
+        this(url, sqlTemplate, true, SqlScriptReader.QUERY_ENDS, null);
     }
 
     public SqlScript(URL url, ISqlTemplate sqlTemplate, boolean failOnError) {
-        this(url, sqlTemplate, failOnError, QUERY_ENDS, null);
+        this(url, sqlTemplate, failOnError, SqlScriptReader.QUERY_ENDS, null);
     }
 
     public SqlScript(URL url, ISqlTemplate sqlTemplate, String delimiter) {
@@ -74,92 +62,49 @@ public SqlScript(URL url, ISqlTemplate sqlTemplate, boolean failOnError, String
         try {
             String fileName = url.getFile();
             fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
-            init(IOUtils.readLines(new InputStreamReader(url.openStream(), "UTF-8")), sqlTemplate,
-                    failOnError, delimiter, replacementTokens);
+            init(new InputStreamReader(url.openStream(), "UTF-8"), sqlTemplate, failOnError,
+                    delimiter, replacementTokens);
         } catch (IOException ex) {
             throw new RuntimeException(ex);
         }
     }
 
     public SqlScript(String sqlScript, ISqlTemplate sqlTemplate, boolean failOnError) {
-        this(sqlScript, sqlTemplate, failOnError, QUERY_ENDS, null);
+        this(sqlScript, sqlTemplate, failOnError, SqlScriptReader.QUERY_ENDS, null);
     }
 
     public SqlScript(String sqlScript, ISqlTemplate sqlTemplate, boolean failOnError,
             String delimiter, Map<String, String> replacementTokens) {
-        try {
-            init(IOUtils.readLines(new StringReader(sqlScript)), sqlTemplate, failOnError,
-                    delimiter, replacementTokens);
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
+        init(new StringReader(sqlScript), sqlTemplate, failOnError, delimiter, replacementTokens);
     }
 
-    public SqlScript(List<String> sqlScript, ISqlTemplate sqlTemplate, boolean failOnError,
+    public SqlScript(Reader reader, ISqlTemplate sqlTemplate, boolean failOnError,
             String delimiter, Map<String, String> replacementTokens) {
-        init(sqlScript, sqlTemplate, failOnError, delimiter, replacementTokens);
+        init(reader, sqlTemplate, failOnError, delimiter, replacementTokens);
     }
 
-    private void init(List<String> sqlScript, ISqlTemplate sqlTemplate, boolean failOnError,
+    private void init(Reader reader, ISqlTemplate sqlTemplate, boolean failOnError,
             String delimiter, Map<String, String> replacementTokens) {
-        this.statements = parseLines(sqlScript, replacementTokens);
+        this.scriptReader = new SqlScriptReader(reader);
+        this.scriptReader.setDelimiter(delimiter);
+        this.scriptReader.setReplacementTokens(replacementTokens);
         this.sqlTemplate = sqlTemplate;
         this.failOnError = failOnError;
-        this.delimiter = delimiter;
-    }
-
-    protected List<String> parseLines(List<String> script, Map<String, String> replacementTokens) {
-        List<String> statements = new ArrayList<String>();
-        StringBuilder sql = new StringBuilder();
-        for (String line : script) {
-            line = trimComments(line);
-            if (line.trim().length() > 0) {
-                if (checkStatementEnds(line)) {
-                    if (sql.length() > 0) {
-                        sql.append("\n");
-                    }
-                    sql.append(line.substring(0, line.lastIndexOf(delimiter)).trim());
-                    String toExecute = sql.toString();
-                    if (StringUtils.isNotBlank(lineDeliminator)) {
-                        toExecute = toExecute.replaceAll(lineDeliminator, "\n");
-                    }
-                    toExecute = FormatUtils.replaceTokens(toExecute, replacementTokens, false);
-                    if (StringUtils.isNotBlank(toExecute)) {
-                        statements.add(toExecute.trim());
-                    }
-                    sql.setLength(0);
-                } else {
-                    sql.append("\n");
-                    sql.append(line);
-                }
-            }
-        }
-        return statements;
     }
 
     public long execute() {
         return execute(false);
     }
 
     public long execute(final boolean autoCommit) {
-        return sqlTemplate.update(autoCommit, failOnError, commitRate, resultsListener,
-                statements.toArray(new String[statements.size()]));
-    }
-
-    private String trimComments(String line) {
-        int index = line.indexOf(COMMENT_CHARS_1);
-        if (index >= 0) {
-            line = line.substring(0, index);
-        }
-        index = line.indexOf(COMMENT_CHARS_2);
-        if (index >= 0) {
-            line = line.substring(0, index);
+        try {
+            long count = this.sqlTemplate.update(autoCommit, failOnError, commitRate,
+                    this.resultsListener, this.scriptReader);
+            return count;
+        } finally {
+            IOUtils.closeQuietly(this.scriptReader);
         }
-        return line;
-    }
 
-    private boolean checkStatementEnds(String s) {
-        return s.trim().endsWith("" + delimiter);
     }
 
     public int getCommitRate() {
@@ -171,7 +116,7 @@ public void setCommitRate(int commitRate) {
     }
 
     public void setLineDeliminator(String lineDeliminator) {
-        this.lineDeliminator = lineDeliminator;
+        this.scriptReader.setDelimiter(lineDeliminator);
     }
 
     public void setListener(ISqlResultsListener listener) {