From 992535fb428c63492c27331b097c5d720db77870 Mon Sep 17 00:00:00 2001 From: Kirill Liubun Date: Mon, 8 Aug 2016 19:25:24 +0300 Subject: [PATCH] Fixed Configuration CTL migration, refactoring and added default record generation --- .../data-migration-0.9.0-0.10.0/pom.xml | 4 ++ .../CTLConfigurationMigration.java | 71 ++++++++++++------- .../data_migration/utils/Constants.java | 10 +++ .../data_migration/utils/DataSources.java | 10 +-- .../data_migration/utils/Utils.java | 18 +++++ .../main/resources/doc/constraint_update.sql | 5 -- .../doc/constraint_update_before.sql | 14 ++++ .../backup-0.9.0.sql} | 2 +- 8 files changed, 98 insertions(+), 36 deletions(-) create mode 100644 server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Constants.java create mode 100644 server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Utils.java delete mode 100644 server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update.sql create mode 100644 server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update_before.sql rename server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/{kaa_old_db.sql => fulldump/backup-0.9.0.sql} (99%) diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/pom.xml b/server/upgrade/data-migration-0.9.0-0.10.0/pom.xml index 3f42a3b210..e427dc3bfa 100644 --- a/server/upgrade/data-migration-0.9.0-0.10.0/pom.xml +++ b/server/upgrade/data-migration-0.9.0-0.10.0/pom.xml @@ -49,6 +49,10 @@ org.kaaproject.kaa.server.common admin-rest-client + + org.kaaproject.kaa.common + core + diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/CTLConfigurationMigration.java b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/CTLConfigurationMigration.java index aa28a616c1..fa8206e395 100644 --- a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/CTLConfigurationMigration.java +++ b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/CTLConfigurationMigration.java @@ -13,10 +13,16 @@ import org.kaaproject.data_migration.model.ConfigurationSchema; import org.kaaproject.data_migration.model.Ctl; import org.kaaproject.data_migration.model.CtlMetaInfo; +import org.kaaproject.data_migration.utils.Utils; +import org.kaaproject.kaa.common.dto.Util; import org.kaaproject.kaa.common.dto.ctl.CTLSchemaDto; import org.kaaproject.kaa.server.common.admin.AdminClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.kaaproject.kaa.server.common.core.algorithms.generation.ConfigurationGenerationException; +import org.kaaproject.kaa.server.common.core.algorithms.generation.DefaultRecordGenerationAlgorithm; +import org.kaaproject.kaa.server.common.core.algorithms.generation.DefaultRecordGenerationAlgorithmImpl; +import org.kaaproject.kaa.server.common.core.configuration.RawData; +import org.kaaproject.kaa.server.common.core.configuration.RawDataFactory; +import org.kaaproject.kaa.server.common.core.schema.RawSchema; import java.io.IOException; @@ -27,6 +33,8 @@ import static java.util.Arrays.asList; import static java.util.stream.Collectors.joining; +import static org.kaaproject.data_migration.utils.Constants.HOST; +import static org.kaaproject.data_migration.utils.Constants.PORT; public class CTLConfigurationMigration { // private static final Logger LOG = LoggerFactory.getLogger(CTLConfigurationMigration.class.getSimpleName()); @@ -36,10 +44,16 @@ public class CTLConfigurationMigration { private Connection connection; private final int NUM_OF_BASE_SCHEMA_FIELDS = 8; - private AdminClient client = new AdminClient("localhost", 8080); + private AdminClient client; public CTLConfigurationMigration(Connection connection) { this.connection = connection; + this.client = new AdminClient(HOST, PORT); + } + + public CTLConfigurationMigration(Connection connection, String host, int port) { + this.connection = connection; + this.client = new AdminClient(host, port); } public void transform() throws SQLException, IOException { @@ -47,7 +61,10 @@ public void transform() throws SQLException, IOException { QueryRunner runner = new QueryRunner(); try { - List schemas = runner.query(connection, "select conf.id as id, created_time as createdTime, created_username as createdUsername, description, name, schems, version, application_id as appId " + +// Utils.runFile(runner, connection, "doc/constraint_update_before.sql"); + + List schemas = runner.query(connection, + "select conf.id as id, created_time as createdTime, created_username as createdUsername, description, name, schems, version, application_id as appId " + "from configuration_schems conf join schems s on conf.id = s.id", new BeanListHandler(ConfigurationSchema.class)); String toDelete = schemas.stream().map(s -> s.getId().toString()).collect(joining(", ")); @@ -58,12 +75,24 @@ public void transform() throws SQLException, IOException { schemas.forEach(s -> s.setId(s.getId() + shift)); Map> confSchemasToCTL = new HashMap<>(); + Long currentCTLMetaId = runner.query(connection, "select max(id) as max_id from ctl_metainfo", rs -> rs.next() ? rs.getLong("max_id") : null); + Long currentCtlId = runner.query(connection, "select max(id) as max_id from ctl", rs -> rs.next() ? rs.getLong("max_id") : null); // CTL creation for (ConfigurationSchema schema : schemas) { + currentCTLMetaId++; + currentCtlId++; + Schema schemaBody = new Schema.Parser().parse(schema.getSchems()); + String fqn = schemaBody.getFullName(); + RawSchema rawSchema = new RawSchema(schemaBody.toString()); + DefaultRecordGenerationAlgorithm algotithm = new DefaultRecordGenerationAlgorithmImpl<>(rawSchema, new RawDataFactory()); + String defaultRecord = algotithm.getRootData().getRawData(); Long tenantId = runner.query(connection, "select tenant_id from application where id = " + schema.getAppId(), rs -> rs.next() ? rs.getLong("tenant_id") : null); - CTLSchemaDto ctlSchemaDto = client.saveCTLSchemaWithAppToken(schema.getSchems(), tenantId.toString(), schema.getAppId().toString()); + runner.insert(connection, "insert into ctl_metainfo values(?, ?, ?, ?)", rs -> null, currentCTLMetaId, fqn, schema.getAppId(), tenantId); + runner.insert(connection, "insert into ctl values(?, ?, ?, ?, ?, ?, ?)", rs -> null, currentCtlId, schema.getSchems(), schema.getCreatedTime(), + schema.getCreatedUsername(), defaultRecord, schema.getVersion(), currentCTLMetaId); + // aggregate configuration schemas with same fqn - Ctl ctl = new Ctl(Long.parseLong(ctlSchemaDto.getId()), new CtlMetaInfo(ctlSchemaDto.getMetaInfo().getFqn(), schema.getAppId(), tenantId)); + Ctl ctl = new Ctl(currentCtlId, new CtlMetaInfo(fqn, schema.getAppId(), tenantId)); if (confSchemasToCTL.containsKey(ctl)) { List list = confSchemasToCTL.get(ctl); list.add(schema); @@ -90,8 +119,8 @@ public void transform() throws SQLException, IOException { } runner.batch(connection, "insert into base_schems values(?, ?, ?, ?, ?, ?, ?, ?)", params.toArray(new Object[schemas.size()][])); - } catch (SQLException e) { - DbUtils.rollback(connection); + } catch (SQLException | ConfigurationGenerationException | IOException e) { + System.err.println("Error: " + e.getMessage()); } finally { DbUtils.closeQuietly(connection); } @@ -102,24 +131,16 @@ private void updateUuids() throws SQLException, IOException { QueryRunner run = new QueryRunner(); ResultSetHandler> rsHandler = new BeanListHandler(Configuration.class); - try { - List configs = run.query(this.connection, "SELECT * FROM configuration", rsHandler); - for (Configuration config : configs) { - JsonNode json = new ObjectMapper().readTree(config.getConfiguration_body()); - JsonNode jsonEncoded = encodeUuids(json); - byte[] encodedConfigurationBody = jsonEncoded.toString().getBytes(); - - int updates = run.update(this.connection, "UPDATE configuration SET configuration_body=? WHERE id=?", encodedConfigurationBody,config.getId()); - if (updates != 1) { -// LOG.error("Failed to update configuration: {}", config); - } else { -// LOG.info("Updated configuration: {}", config); - } + List configs = run.query(this.connection, "SELECT * FROM configuration", rsHandler); + for (Configuration config : configs) { + JsonNode json = new ObjectMapper().readTree(config.getConfiguration_body()); + JsonNode jsonEncoded = encodeUuids(json); + byte[] encodedConfigurationBody = jsonEncoded.toString().getBytes(); + + int updates = run.update(this.connection, "UPDATE configuration SET configuration_body=? WHERE id=?", encodedConfigurationBody,config.getId()); + if (updates == 1) { + //TODO } - } catch (SQLException e) { -// LOG.error("Failed to load configurations. {}", e); - } finally { - DbUtils.close(this.connection); } } diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Constants.java b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Constants.java new file mode 100644 index 0000000000..be7859ee90 --- /dev/null +++ b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Constants.java @@ -0,0 +1,10 @@ +package org.kaaproject.data_migration.utils; + + +final public class Constants { + public static final String USER_NAME = "sqladmin"; + public static final String PASSWORD = "admin"; + public static final String DB_NAME = "kaa"; + public static final String HOST = "10.2.1.130"; + public static final Integer PORT = 8080; +} diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/DataSources.java b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/DataSources.java index 899b8cdb66..b577701b17 100644 --- a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/DataSources.java +++ b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/DataSources.java @@ -5,14 +5,14 @@ import javax.sql.DataSource; +import static org.kaaproject.data_migration.utils.Constants.*; + public enum DataSources { MARIADB(getMariaDB()), POSTGRES(getPostgreSQL()); private final DataSource ds; - private static final String USER_NAME = "root"; - private static final String PASSWORD = "kaa"; - private static final String DB_NAME = "kaa"; + DataSources(DataSource ds) { this.ds = ds; @@ -25,7 +25,7 @@ public DataSource getDs() { private static DataSource getPostgreSQL() { BasicDataSource bds = new BasicDataSource(); bds.setDriverClassName("org.postgresql.Driver"); - bds.setUrl("jdbc:postgresql://localhost:5432/" + DB_NAME); + bds.setUrl("jdbc:postgresql://" + HOST +":5432/" + DB_NAME); bds.setUsername(USER_NAME); bds.setPassword(PASSWORD); return bds; @@ -35,7 +35,7 @@ private static DataSource getPostgreSQL() { private static DataSource getMariaDB() { BasicDataSource bds = new BasicDataSource(); bds.setDriverClassName("org.mariadb.jdbc.Driver"); - bds.setUrl("jdbc:mysql://localhost:3306/" + DB_NAME); + bds.setUrl("jdbc:mysql://" + HOST +":3306/" + DB_NAME); bds.setUsername(USER_NAME); bds.setPassword(PASSWORD); // bds.setDefaultAutoCommit(false); diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Utils.java b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Utils.java new file mode 100644 index 0000000000..703ea0b6fa --- /dev/null +++ b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/java/org/kaaproject/data_migration/utils/Utils.java @@ -0,0 +1,18 @@ +package org.kaaproject.data_migration.utils; + + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.io.IOUtils; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +public class Utils { + + public static void runFile(QueryRunner runner, Connection connection, String fileName) throws IOException, SQLException { + String query = IOUtils.toString(Utils.class.getClassLoader().getResourceAsStream(fileName)); + runner.update(connection, query); + } + +} diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update.sql b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update.sql deleted file mode 100644 index a7aa4c3373..0000000000 --- a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE configuration_schems DROP FOREIGN KEY FK_6c1w1hvw3794uenuce20voli9; - -ALTER TABLE configuration DROP FOREIGN KEY FK_smcmof238to6x3ta7enujgd8m; -ALTER TABLE configuration add constraint `FK_configuration_schems_id` -FOREIGN KEY (`configuration_schems_id`) REFERENCES `configuration_schems` (`id`) ON DELETE CASCADE ON UPDATE CASCADE; \ No newline at end of file diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update_before.sql b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update_before.sql new file mode 100644 index 0000000000..98418bfc39 --- /dev/null +++ b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/constraint_update_before.sql @@ -0,0 +1,14 @@ +SET @const_name = (SELECT CONSTRAINT_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA = 'kaa' AND TABLE_NAME = 'configuration_schems' and referenced_table_name='schems'); +SET @sql = CONCAT('ALTER TABLE configuration_schems DROP FOREIGN KEY ', @const_name); +PREPARE s from @sql; +EXECUTE s; +DEALLOCATE PREPARE s; + +SET @const_name = (SELECT CONSTRAINT_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA = 'kaa' AND TABLE_NAME = 'configuration' and referenced_table_name='configuration_schems'); +SET @sql = CONCAT('ALTER TABLE configuration DROP FOREIGN KEY ', @const_name); +PREPARE s from @sql; +EXECUTE s; +DEALLOCATE PREPARE s; + +ALTER TABLE configuration add constraint `FK_configuration_schems_id` +FOREIGN KEY (`configuration_schems_id`) REFERENCES `configuration_schems` (`id`) ON DELETE CASCADE ON UPDATE CASCADE; \ No newline at end of file diff --git a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/kaa_old_db.sql b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/fulldump/backup-0.9.0.sql similarity index 99% rename from server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/kaa_old_db.sql rename to server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/fulldump/backup-0.9.0.sql index 55e08858a4..d9b644000b 100644 --- a/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/kaa_old_db.sql +++ b/server/upgrade/data-migration-0.9.0-0.10.0/src/main/resources/doc/fulldump/backup-0.9.0.sql @@ -1081,4 +1081,4 @@ UNLOCK TABLES; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2016-08-01 3:23:12 +-- Dump completed on 2016-08-08 5:55:48