diff --git a/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbDump.java b/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbDump.java index a3545532f4..e19de32b83 100644 --- a/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbDump.java +++ b/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbDump.java @@ -21,10 +21,12 @@ package org.jumpmind.symmetric; +import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import org.apache.commons.cli.CommandLine; @@ -32,10 +34,18 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.jumpmind.db.io.DatabaseIO; +import org.jumpmind.db.model.Column; import org.jumpmind.db.model.Database; +import org.jumpmind.db.model.Table; import org.jumpmind.db.platform.DatabasePlatformSettings; import org.jumpmind.db.platform.IDatabasePlatform; import org.jumpmind.db.platform.mysql.MySqlPlatform; +import org.jumpmind.db.sql.DmlStatement; +import org.jumpmind.db.sql.DmlStatement.DmlType; +import org.jumpmind.db.sql.ISqlRowMapper; +import org.jumpmind.db.sql.ISqlTemplate; +import org.jumpmind.db.sql.Row; +import org.jumpmind.symmetric.csv.CsvWriter; /** * Dump the structure and data from database tables to file. @@ -46,7 +56,9 @@ public class DbDump extends AbstractCommandLauncher { private static final Log log = LogFactory.getLog(DbDump.class); private static final String OPTION_XML = "xml"; - + + private static final String OPTION_CSV = "csv"; + private static final String OPTION_COMPATIBLE = "compatible"; private static final String OPTION_ADD_DROP_TABLE = "add-drop-table"; @@ -75,6 +87,7 @@ protected void printHelp(Options options) { protected void buildOptions(Options options) { super.buildOptions(options); addOption(options, "x", OPTION_XML, false); + addOption(options, null, OPTION_CSV, false); addOption(options, null, OPTION_COMPATIBLE, true); addOption(options, null, OPTION_ADD_DROP_TABLE, false); addOption(options, null, OPTION_NO_CREATE_INFO, false); @@ -87,31 +100,50 @@ protected boolean executeOptions(CommandLine line) throws Exception { // TODO: get table names list as args if (line.hasOption(OPTION_XML)) { - dumpSchemaAsXml(System.out); + dumpTablesAsXml(System.out, line.getArgs(), line.hasOption(OPTION_NO_CREATE_INFO), + line.hasOption(OPTION_NO_DATA), line.hasOption(OPTION_COMMENTS)); + } else if (line.hasOption(OPTION_CSV)) { + dumpTablesAsCsv(System.out, line.getArgs(), line.hasOption(OPTION_NO_DATA), line.hasOption(OPTION_COMMENTS)); } else { - dumpSchemaAsSql(System.out, line.hasOption(OPTION_ADD_DROP_TABLE), line.hasOption(OPTION_NO_CREATE_INFO), + dumpTablesAsSql(System.out, line.getArgs(), line.hasOption(OPTION_ADD_DROP_TABLE), line.hasOption(OPTION_NO_CREATE_INFO), line.hasOption(OPTION_NO_DATA), line.hasOption(OPTION_COMMENTS)); } return true; } - public void dumpSchemaAsXml(OutputStream output) throws Exception { + public void dumpTablesAsXml(OutputStream output, String[] tables, boolean noCreateInfo, boolean noData, boolean comments) throws Exception { /* TODO: * * value */ IDatabasePlatform platform = getDatabasePlatform(); - Database db = platform.readDatabase(platform.getDefaultCatalog(), platform.getDefaultSchema(), null); + String catalog = platform.getDefaultCatalog(); + String schema = platform.getDefaultSchema(); + + Database db = platform.readDatabase(catalog, schema, null); + Writer writer = new OutputStreamWriter(output); + SimpleDateFormat df = new SimpleDateFormat("YYYY-MM-dd HH:mm:ss"); + + if (comments) { + writer.write("\n"); + writer.write("\n"); + writer.write("\n"); + } + new DatabaseIO().write(db, output); - output.flush(); + writer.flush(); + writer.close(); } - public void dumpSchemaAsSql(OutputStream output, boolean addDropTable, boolean noCreateInfo, boolean noData, boolean comments) throws Exception { + public void dumpTablesAsSql(OutputStream output, String[] tables, boolean addDropTable, boolean noCreateInfo, boolean noData, boolean comments) throws Exception { IDatabasePlatform platform = getDatabasePlatform(); String catalog = platform.getDefaultCatalog(); String schema = platform.getDefaultSchema(); Database db = platform.readDatabase(catalog, schema, null); + + // IDatabasePlatform target = Factory.getPlatform("mysql"); + IDatabasePlatform target = new MySqlPlatform(null, new DatabasePlatformSettings()); Writer writer = new OutputStreamWriter(output); SimpleDateFormat df = new SimpleDateFormat("YYYY-MM-dd HH:mm:ss"); @@ -138,6 +170,64 @@ public void dumpSchemaAsSql(OutputStream output, boolean addDropTable, boolean n writer.close(); } + public void dumpTablesAsCsv(OutputStream output, String[] tableNames, boolean noData, boolean comments) throws Exception { + final IDatabasePlatform platform = getDatabasePlatform(); + Writer writer = new OutputStreamWriter(output); + String catalog = platform.getDefaultCatalog(); + String schema = platform.getDefaultSchema(); + ArrayList tableList = new ArrayList
(); + + if (tableNames.length == 0) { + Database database = platform.readDatabase(catalog, schema, null); + for (Table table : database.getTables()) { + tableList.add(table); + } + } else { + for (String tableName : tableNames) { + Table table = platform.readTableFromDatabase(catalog, schema, tableName); + if (table != null) { + tableList.add(table); + } else { + throw new RuntimeException("Cannot find table " + tableName + " in catalog " + catalog + + " and schema " + schema); + } + } + } + + for (Table table : tableList) { + final CsvWriter csvWriter = new CsvWriter(writer, ','); + csvWriter.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH); + + if (comments) { + csvWriter.writeComment(" Table: " + table.getFullyQualifiedTableName()); + } + + csvWriter.writeRecord(table.getColumnNames()); + + if (!noData) { + ISqlTemplate sqlTemplate = platform.getSqlTemplate(); + DmlStatement stmt = platform.createDmlStatement(DmlType.SELECT_ALL, table); + final Column[] columns = table.getColumns(); + + sqlTemplate.queryForObject(stmt.getSql(), new ISqlRowMapper() { + public Object mapRow(Row row) { + String[] values = platform.getStringValues(columns, row); + try { + csvWriter.writeRecord(values, true); + } catch (IOException e) { + throw new RuntimeException(e); + } + return values; + } + }); + } + csvWriter.flush(); + } + + writer.flush(); + writer.close(); + } + // public void copyFromTables(List tables) { // long batchId = 1; // for (TableToExtract tableToRead : tables) { diff --git a/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbLoad.java b/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbLoad.java new file mode 100644 index 0000000000..b2f65b3533 --- /dev/null +++ b/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/DbLoad.java @@ -0,0 +1,154 @@ +/* + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Lesser General Public License (the + * "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jumpmind.symmetric; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.jumpmind.db.io.DatabaseIO; +import org.jumpmind.db.model.Database; +import org.jumpmind.db.model.Table; +import org.jumpmind.db.platform.IDatabasePlatform; +import org.jumpmind.db.sql.DmlStatement; +import org.jumpmind.db.sql.DmlStatement.DmlType; +import org.jumpmind.db.sql.ISqlTemplate; +import org.jumpmind.db.sql.SqlScript; +import org.jumpmind.db.util.BinaryEncoding; +import org.jumpmind.symmetric.csv.CsvReader; + +/** + * Load data from file to database tables. + */ +public class DbLoad extends AbstractCommandLauncher { + + @SuppressWarnings("unused") + private static final Log log = LogFactory.getLog(DbLoad.class); + + private static final String OPTION_XML = "xml"; + + private static final String OPTION_CSV = "csv"; + + public DbLoad(String commandName, String messageKeyPrefix) { + super(commandName, messageKeyPrefix); + } + + public static void main(String[] args) throws Exception { + new DbLoad("dbload", "DbLoad.Option.").execute(args); + } + + protected void printHelp(Options options) { + System.out.println(commandName + " version " + Version.version()); + System.out.println("Load data from file to database tables.\n"); + super.printHelp(options); + } + + @Override + protected void buildOptions(Options options) { + super.buildOptions(options); + addOption(options, "x", OPTION_XML, false); + addOption(options, null, OPTION_CSV, false); + } + + @Override + protected boolean executeOptions(CommandLine line) throws Exception { + String[] args = line.getArgs(); + + if (args.length == 0) { + executeOption(line, System.in); + } else { + for (String fileName : args) { + if (! new File(fileName).exists()) { + throw new RuntimeException("Cannot find file " + fileName); + } + } + for (String fileName : args) { + executeOption(line, new FileInputStream(fileName)); + } + } + + return true; + } + + protected void executeOption(CommandLine line, InputStream in) throws Exception { + if (line.hasOption(OPTION_XML)) { + loadTablesFromXml(in); + } else if (line.hasOption(OPTION_CSV)) { + loadTablesFromCsv(in); + } else { + loadTablesFromSql(in); + } + } + + public void loadTablesFromXml(InputStream in) throws Exception { + // TODO: read in data from XML also + IDatabasePlatform platform = getDatabasePlatform(); + Database database = new DatabaseIO().read(in); + platform.createDatabase(database, false, true); + } + + public void loadTablesFromCsv(InputStream in) throws Exception { + IDatabasePlatform platform = getDatabasePlatform(); + ISqlTemplate sqlTemplate = platform.getSqlTemplate(); + Table table = platform.readTableFromDatabase(platform.getDefaultCatalog(), platform.getDefaultSchema(), "item"); + if (table == null) { + throw new RuntimeException("Unable to find table"); + } + DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, table); + + CsvReader csvReader = new CsvReader(new InputStreamReader(in)); + csvReader.setEscapeMode(CsvReader.ESCAPE_MODE_BACKSLASH); + csvReader.setSafetySwitch(false); + csvReader.setUseComments(true); + csvReader.readHeaders(); + + while (csvReader.readRecord()) { + String[] values = csvReader.getValues(); + Object[] data = platform.getObjectValues(BinaryEncoding.HEX, table, csvReader.getHeaders(), values); + for (String value : values) { + System.out.print("|" + value); + } + System.out.print("\n"); + int rows = sqlTemplate.update(statement.getSql(), data); + System.out.println(rows + " rows updated."); + } + csvReader.close(); + } + + public void loadTablesFromSql(InputStream in) throws Exception { + IDatabasePlatform platform = getDatabasePlatform(); + + // TODO: SqlScript should be able to stream from standard input to run large SQL script + List lines = IOUtils.readLines(in); + + SqlScript script = new SqlScript(lines, platform.getSqlTemplate(), true, SqlScript.QUERY_ENDS, + platform.getSqlScriptReplacementTokens()); + script.execute(); + } +} \ No newline at end of file diff --git a/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/SymmetricAdmin.java b/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/SymmetricAdmin.java index 40873b31e7..455e4722ec 100644 --- a/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/SymmetricAdmin.java +++ b/symmetric/symmetric-client/src/main/java/org/jumpmind/symmetric/SymmetricAdmin.java @@ -24,12 +24,10 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; -import java.net.MalformedURLException; import java.nio.charset.Charset; import org.apache.commons.cli.CommandLine; @@ -37,12 +35,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.jumpmind.db.io.DatabaseIO; -import org.jumpmind.db.model.Database; -import org.jumpmind.db.platform.IDatabasePlatform; -import org.jumpmind.db.sql.SqlScript; import org.jumpmind.symmetric.common.SecurityConstants; -import org.jumpmind.symmetric.db.ISymmetricDialect; import org.jumpmind.symmetric.service.IDataExtractorService; import org.jumpmind.symmetric.service.IDataLoaderService; import org.jumpmind.symmetric.service.IDataService; @@ -76,10 +69,6 @@ public class SymmetricAdmin extends AbstractCommandLauncher { private static final String OPTION_PURGE = "purge"; - private static final String OPTION_RUN_DDL_XML = "run-ddl"; - - private static final String OPTION_RUN_SQL = "run-sql"; - private static final String OPTION_PROPERTIES_GEN = "generate-default-properties"; private static final String OPTION_LOAD_BATCH = "load-batch"; @@ -105,16 +94,20 @@ protected void printHelp(Options options) { @Override protected void buildOptions(Options options) { super.buildOptions(options); - addOption(options, "c", OPTION_DDL_GEN, true); + + // TODO: move generate symmetric DDL to DbDump? + addOption(options, "c", OPTION_DDL_GEN, true); + addOption(options, "X", OPTION_PURGE, false); addOption(options, "g", OPTION_PROPERTIES_GEN, true); - addOption(options, "r", OPTION_RUN_DDL_XML, true); - addOption(options, "s", OPTION_RUN_SQL, true); addOption(options, "a", OPTION_AUTO_CREATE, false); addOption(options, "R", OPTION_OPEN_REGISTRATION, true); addOption(options, "l", OPTION_RELOAD_NODE, true); + + // TODO: move dump batch and load batch to DbDump/Load? addOption(options, "d", OPTION_DUMP_BATCH, true); addOption(options, "b", OPTION_LOAD_BATCH, true); + addOption(options, "t", OPTION_TRIGGER_GEN, true); addOption(options, "o", OPTION_TRIGGER_GEN_ALWAYS, false); addOption(options, "e", OPTION_ENCRYPT_TEXT, true); @@ -193,18 +186,6 @@ protected boolean executeOptions(CommandLine line) throws Exception { return true; } - if (line.hasOption(OPTION_RUN_DDL_XML)) { - runDdlXml(getSymmetricEngine(), line.getOptionValue(OPTION_RUN_DDL_XML)); - System.exit(0); - return true; - } - - if (line.hasOption(OPTION_RUN_SQL)) { - runSql(getSymmetricEngine(), line.getOptionValue(OPTION_RUN_SQL)); - System.exit(0); - return true; - } - if (line.hasOption(OPTION_LOAD_BATCH)) { loadBatch(getSymmetricEngine(), line.getOptionValue(OPTION_LOAD_BATCH)); System.exit(0); @@ -329,30 +310,4 @@ private void autoCreateDatabase(ISymmetricEngine engine) { engine.setupDatabase(true); } - private void runDdlXml(ISymmetricEngine engine, String fileName) throws FileNotFoundException { - ISymmetricDialect dialect = engine.getSymmetricDialect(); - File file = new File(fileName); - if (file.exists() && file.isFile()) { - IDatabasePlatform pf = dialect.getPlatform(); - Database db = new DatabaseIO().read(new File(fileName)); - pf.createDatabase(db, false, true); - } else { - throw new SymmetricException("Could not find file %s", fileName); - } - } - - private void runSql(ISymmetricEngine engine, String fileName) throws FileNotFoundException, - MalformedURLException { - ISymmetricDialect dialect = engine.getSymmetricDialect(); - File file = new File(fileName); - if (file.exists() && file.isFile()) { - SqlScript script = new SqlScript(file.toURI().toURL(), dialect.getPlatform() - .getSqlTemplate(), true, SqlScript.QUERY_ENDS, dialect.getPlatform() - .getSqlScriptReplacementTokens()); - script.execute(); - } else { - throw new SymmetricException("Could not find file %s", fileName); - } - } - } diff --git a/symmetric/symmetric-client/src/main/resources/symmetric-messages.properties b/symmetric/symmetric-client/src/main/resources/symmetric-messages.properties index af47febe90..e8d1567668 100644 --- a/symmetric/symmetric-client/src/main/resources/symmetric-messages.properties +++ b/symmetric/symmetric-client/src/main/resources/symmetric-messages.properties @@ -28,8 +28,6 @@ Launcher.Option.no-directbuffer=Do not use direct buffers for the NIO HTTP conne SymAdmin.Option.generate-config-ddl=Output the DDL to create the symmetric tables. Takes an argument of the name of the file to write the ddl to. SymAdmin.Option.purge=Will simply run the purge process against the currently configured database. SymAdmin.Option.generate-default-properties=Takes an argument with the path to a file which all the default overrideable properties will be written. -SymAdmin.Option.run-ddl=Takes an argument of a DdlUtils xml file and applies it to the database configured in your symmetric properties file. -SymAdmin.Option.run-sql=Takes an argument of a .sql file and runs it against the database configured in your symmetric properties file. SymAdmin.Option.run-profile=Takes an argument of a profile name. When run, SymmetricDS will be configured per the profile setup. A profile is a SymmetricDS configuration for a specific application, topology, or domain. SymAdmin.Option.auto-create=Attempts to create the symmetric tables in the configured database. SymAdmin.Option.open-registration=Open registration for the passed in node group and external id. Takes an argument of {groupId},{externalId}. @@ -39,7 +37,6 @@ SymAdmin.Option.load-batch=Load the CSV contents of the specified file. SymAdmin.Option.generate-triggers=Run the sync triggers process and write the output the specified file. If triggers should not be applied automatically then set the auto.sync.triggers property to false SymAdmin.Option.generate-triggers-always=Run the sync triggers process even if the triggers already exist. SymAdmin.Option.encrypt=Encrypts the given text for use with db.user and db.password properties. -SymAdmin.Option.export-schema=Export the entire database schema for the configured database connection to XML. Takes an argument of the file name. SymAdmin.Option.create-war=Generate a web archive that can be deployed to a web server like Tomcat. The name of the output file must be provided. If a properties file is designated, it will be renamed and packaged as symmetric.properties. Other than the optional properties file, a war is made up of the contents of the web directory and the conf directory of the standalone installation. DbDump.Option.compatible=Change dump to be compatible with given database: db2, derby, firebird, greenplum, h2, hsqldb, hsqldb2, informix, interbase, mssql, mysql, oracle, postgresql, sybase. @@ -48,3 +45,4 @@ DbDump.Option.no-create-info=Do not write statements to create tables. DbDump.Option.no-data=Do not write statements to insert into tables. DbDump.Option.comments=Write informational comments. DbDump.Option.xml=Write output as XML. +DbDump.Option.csv=Write output as CSV.