From 8e34ce8788b3dda23ae370560db6b6b5121aafca Mon Sep 17 00:00:00 2001 From: Lucas Laurent Date: Mon, 4 Mar 2024 15:01:34 +0100 Subject: [PATCH 1/6] Add action that imports init data of a package from a database tables data. --- packages/core/actions/init/import.php | 312 ++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 packages/core/actions/init/import.php diff --git a/packages/core/actions/init/import.php b/packages/core/actions/init/import.php new file mode 100644 index 000000000..657f4fc2b --- /dev/null +++ b/packages/core/actions/init/import.php @@ -0,0 +1,312 @@ + + Some Rights Reserved, Cedric Francoys, 2010-2021 + Licensed under GNU LGPL 3 license +*/ + +use equal\data\adapt\DataAdapter; +use equal\db\DBConnection; +use equal\db\DBManipulator; +use equal\db\DBManipulatorMySQL; +use equal\db\DBManipulatorSQLite; +use equal\db\DBManipulatorSqlSrv; + +list($params, $providers) = eQual::announce([ + 'description' => 'Import data from a database to eQual database for a given package, needs a configuration file init/import-config.json in folder of given package.', + 'params' => [ + 'db_dbms' => [ + 'type' => 'string', + 'default' => 'MYSQL' + ], + + 'db_host' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_port' => [ + 'type' => 'integer', + 'required' => true + ], + + 'db_user' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_password' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_name' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_charset' => [ + 'type' => 'string', + 'default' => 'utf8mb4' + ], + + 'db_collation' => [ + 'type' => 'string', + 'default' => 'utf8mb4_unicode_ci' + ], + + 'package' => [ + 'type' => 'string', + 'required' => true + ], + + 'entity' => [ + 'type' => 'string' + ] + ], + 'response' => [ + 'content-type' => 'application/json', + 'charset' => 'utf-8', + 'accept-origin' => '*' + ], + 'providers' => ['context', 'adapt'] +]); + +/** + * @var \equal\php\Context $context + * @var \equal\data\DataAdapterProvider $dap + */ +list($context, $dap) = [$providers['context'], $providers['adapt']]; + +/** @var $adapter DataAdapter */ +$adapter = $dap->get('sql'); + +$import_config = json_decode( + file_get_contents(QN_BASEDIR . '/packages/' . $params['package'] . '/init/import-config.json'), + true +); + +$old_db_connection = createOldDbConnection( + $params['db_dbms'], $params['db_host'], $params['db_port'], $params['db_name'], + $params['db_user'], $params['db_password'], $params['db_charset'], $params['db_collation'] +); + +$new_db_connection = createNewDbConnection(); + +$limit = 500; + +foreach($import_config as $config) { + if(isset($params['entity']) && $params['entity'] !== $config['entity']) { + continue; + } + + $new_table_name = str_replace('\\', '_', $config['entity']); + + $offset = 0; + $remaining_data = true; + while($remaining_data) { + $res = $old_db_connection->getRecords( + $config['old_table']['name'], + array_keys($config['old_table']['fields']), + null, + $config['old_table']['conditions'] ?? null, + $config['old_table']['id_field'] ?? 'id', + [], + $offset, + $limit + ); + + if($res->num_rows < $limit) { + $remaining_data = false; + } + + $items = []; + while ($row = $old_db_connection->fetchArray($res)) { + $old_item = castOldDbRow($row, $config['old_table']['fields']); + $items[] = createNewItemFromOld($config, $old_db_connection, $old_item); + } + + if(!empty($items)) { + foreach($items as &$item) { + $item['created'] = $adapter->adaptOut($item['created'], 'datetime'); + $item['modified'] = $adapter->adaptOut($item['modified'], 'datetime'); + } + + $new_db_connection->addRecords( + $new_table_name, + array_keys($items[0]), + $items + ); + } + + $offset += $limit; + } +} + +$old_db_connection->disconnect(); +$new_db_connection->disconnect(); + +$context->httpResponse() + ->body(['success' => true]) + ->send(); + + +function createOldDbConnection(string $dbms, string $host, int $port, string $name, string $user, string $password, string $charset, string $collation) { + $db_manipulator_map = [ + 'MARIADB' => DBManipulatorMySQL::class, + 'MYSQL' => DBManipulatorMySQL::class, + 'SQLSRV' => DBManipulatorSqlSrv::class, + 'SQLITE' => DBManipulatorSQLite::class, + ]; + + if(!isset($db_manipulator_map[$dbms])) { + throw new Exception('DBMS ' . $dbms . ' not handled', QN_ERROR_INVALID_CONFIG); + } + + $db_connection = new $db_manipulator_map[$dbms]($host, $port, $name, $user, $password, $charset, $collation); + + $db_connection->connect(); + if(!$db_connection->connected()) { + throw new Exception( + 'Unable to establish connection to DBMS host (wrong credentials) for old connection', + QN_ERROR_INVALID_CONFIG + ); + } + + return $db_connection; +} + +function createNewDbConnection() { + $db_connection = DBConnection::getInstance( + constant('DB_HOST'), constant('DB_PORT'), constant('DB_NAME'), + constant('DB_USER'), constant('DB_PASSWORD'), + constant('DB_DBMS') + ); + + $db_connection->connect(); + if(!$db_connection->connected()) { + throw new Exception('Unable to establish connection to DBMS host (wrong credentials)', QN_ERROR_INVALID_CONFIG); + } + + return $db_connection; +} + +function castOldDbRow(array $old_item, array $old_table_fields_config): array { + $old_item_casted = []; + foreach($old_item as $key => $column_value) { + switch($old_table_fields_config[$key]) { + case 'integer': + $old_item_casted[$key] = (int) $column_value; + break; + case 'float': + $old_item_casted[$key] = (float) $column_value; + break; + case 'boolean': + $old_item_casted[$key] = (bool) $column_value; + break; + case 'date': + $old_item_casted[$key] = strtotime($column_value); + break; + default: + $old_item_casted[$key] = $column_value; + break; + } + } + + return $old_item_casted; +} + +function createNewItemFromOld(array $config, DBManipulator $old_db_connection, array $old_item): array { + $item = [ + 'creator' => QN_ROOT_USER_ID, + 'created' => time(), + 'modifier' => QN_ROOT_USER_ID, + 'modified' => time(), + 'deleted' => false, + 'state' => 'instance' + ]; + + foreach($config['data_map'] as $new_key => $import_conf) { + if(is_string($import_conf)) { + $import_conf = [ + 'type' => 'field', + 'field' => $import_conf + ]; + } + + $imp_confs = isset($import_conf['type']) ? [$import_conf] : $import_conf; + + $previous_value = $old_item[$imp_confs[0]['field']] ?? null; + foreach($imp_confs as $imp_conf) { + switch($imp_conf['type']) { + case 'value': + $item[$new_key] = $import_conf['value']; + break; + case 'field': + $item[$new_key] = $previous_value; + break; + case 'cast': + switch($imp_conf['cast']) { + case 'integer': + $item[$new_key] = (int) $previous_value; + break; + case 'boolean': + $item[$new_key] = (bool) $previous_value; + break; + case 'string': + $item[$new_key] = $previous_value . ''; + break; + } + break; + case 'round': + $item[$new_key] = round($previous_value); + break; + case 'field-contains': + $item[$new_key] = strpos($previous_value, $imp_conf['value']) !== false; + break; + case 'field-does-not-contain': + $item[$new_key] = strpos($previous_value, $imp_conf['value']) === false; + break; + case 'map-value': + $match_found = false; + foreach($imp_conf['map'] as $map_item) { + if($map_item['old'] != $previous_value) { + continue; + } + + $item[$new_key] = $map_item['new']; + $match_found = true; + break; + } + + if(!$match_found) { + $item[$new_key] = $previous_value; + } + + break; + case 'query': + if(is_null($previous_value)) { + break; + } + + $query = $imp_conf['query']; + $resRel = $old_db_connection->sendQuery( + 'SELECT `' . $query['field'] . '` from `' . $query['table'] . '` WHERE `' . ($query['where_field'] ?? 'id') . '` = ' . $previous_value . ' LIMIT 1;' + ); + + if($relRow = $old_db_connection->fetchArray($resRel)) { + $item[$new_key] = $relRow[$query['field']]; + } + else { + $item[$new_key] = null; + } + + break; + } + + $previous_value = $item[$new_key]; + } + } + + return $item; +} From c643602d3690da07f50a472bb152761d5f9111ba Mon Sep 17 00:00:00 2001 From: Lucas Laurent Date: Mon, 4 Mar 2024 15:22:37 +0100 Subject: [PATCH 2/6] Refacto get import config and throw errors if problem reading config --- packages/core/actions/init/import.php | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/core/actions/init/import.php b/packages/core/actions/init/import.php index 657f4fc2b..d57bf53d7 100644 --- a/packages/core/actions/init/import.php +++ b/packages/core/actions/init/import.php @@ -81,9 +81,8 @@ /** @var $adapter DataAdapter */ $adapter = $dap->get('sql'); -$import_config = json_decode( - file_get_contents(QN_BASEDIR . '/packages/' . $params['package'] . '/init/import-config.json'), - true +$import_config = getImportConfig( + QN_BASEDIR . '/packages/' . $params['package'] . '/init/import-config.json' ); $old_db_connection = createOldDbConnection( @@ -150,6 +149,19 @@ ->body(['success' => true]) ->send(); +function getImportConfig($config_file_path): array { + $config_file_content = file_get_contents($config_file_path); + if(!$config_file_content) { + throw new Exception('Missing import config file ' . $config_file_path, QN_ERROR_INVALID_CONFIG); + } + + $import_config = json_decode($config_file_content, true); + if(!is_array($import_config)) { + throw new Exception('Invalid import configuration file', QN_ERROR_INVALID_CONFIG); + } + + return $import_config; +} function createOldDbConnection(string $dbms, string $host, int $port, string $name, string $user, string $password, string $charset, string $collation) { $db_manipulator_map = [ From 004ce79ce9f3780f505afcd5f9f23e093cc03fae Mon Sep 17 00:00:00 2001 From: Lucas Laurent Date: Mon, 4 Mar 2024 15:34:37 +0100 Subject: [PATCH 3/6] harmonise code --- packages/core/actions/init/import.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/actions/init/import.php b/packages/core/actions/init/import.php index d57bf53d7..69bb23bae 100644 --- a/packages/core/actions/init/import.php +++ b/packages/core/actions/init/import.php @@ -266,7 +266,7 @@ function createNewItemFromOld(array $config, DBManipulator $old_db_connection, a $item[$new_key] = (bool) $previous_value; break; case 'string': - $item[$new_key] = $previous_value . ''; + $item[$new_key] = (string) $previous_value; break; } break; From 15486b0653a3bb32bfdd40bbc5f7e4163fc53087 Mon Sep 17 00:00:00 2001 From: Lucas Laurent Date: Mon, 4 Mar 2024 15:59:58 +0100 Subject: [PATCH 4/6] Add multiply and divide mappers --- packages/core/actions/init/import.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core/actions/init/import.php b/packages/core/actions/init/import.php index 69bb23bae..f02cc9314 100644 --- a/packages/core/actions/init/import.php +++ b/packages/core/actions/init/import.php @@ -273,6 +273,12 @@ function createNewItemFromOld(array $config, DBManipulator $old_db_connection, a case 'round': $item[$new_key] = round($previous_value); break; + case 'multiply': + $item[$new_key] = $previous_value * $import_conf['by']; + break; + case 'divide': + $item[$new_key] = $previous_value / $import_conf['by']; + break; case 'field-contains': $item[$new_key] = strpos($previous_value, $imp_conf['value']) !== false; break; From 7b8912291cf8c42d4810e239fee1eb6bf2b5fb01 Mon Sep 17 00:00:00 2001 From: Lucas Laurent Date: Wed, 6 Mar 2024 16:08:26 +0100 Subject: [PATCH 5/6] Add computed mapper --- packages/core/actions/init/import.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core/actions/init/import.php b/packages/core/actions/init/import.php index f02cc9314..4717e75c9 100644 --- a/packages/core/actions/init/import.php +++ b/packages/core/actions/init/import.php @@ -257,6 +257,12 @@ function createNewItemFromOld(array $config, DBManipulator $old_db_connection, a case 'field': $item[$new_key] = $previous_value; break; + case 'computed': + $item[$new_key] = $imp_conf['value']; + foreach($imp_conf['fields'] as $f) { + $item[$new_key] = str_replace('%'.$f.'%', $old_item[$f], $item[$new_key]); + } + break; case 'cast': switch($imp_conf['cast']) { case 'integer': From 52fed206aaa1c9f7c8adf98230513d2108f77923 Mon Sep 17 00:00:00 2001 From: Lucas Laurent Date: Thu, 14 Mar 2024 10:31:44 +0100 Subject: [PATCH 6/6] action refacto to be extendable --- packages/core/actions/init/import.php | 293 +++++++++++++------------- 1 file changed, 147 insertions(+), 146 deletions(-) diff --git a/packages/core/actions/init/import.php b/packages/core/actions/init/import.php index 4717e75c9..b0b3d86aa 100644 --- a/packages/core/actions/init/import.php +++ b/packages/core/actions/init/import.php @@ -12,144 +12,7 @@ use equal\db\DBManipulatorSQLite; use equal\db\DBManipulatorSqlSrv; -list($params, $providers) = eQual::announce([ - 'description' => 'Import data from a database to eQual database for a given package, needs a configuration file init/import-config.json in folder of given package.', - 'params' => [ - 'db_dbms' => [ - 'type' => 'string', - 'default' => 'MYSQL' - ], - - 'db_host' => [ - 'type' => 'string', - 'required' => true - ], - - 'db_port' => [ - 'type' => 'integer', - 'required' => true - ], - - 'db_user' => [ - 'type' => 'string', - 'required' => true - ], - - 'db_password' => [ - 'type' => 'string', - 'required' => true - ], - - 'db_name' => [ - 'type' => 'string', - 'required' => true - ], - - 'db_charset' => [ - 'type' => 'string', - 'default' => 'utf8mb4' - ], - - 'db_collation' => [ - 'type' => 'string', - 'default' => 'utf8mb4_unicode_ci' - ], - - 'package' => [ - 'type' => 'string', - 'required' => true - ], - - 'entity' => [ - 'type' => 'string' - ] - ], - 'response' => [ - 'content-type' => 'application/json', - 'charset' => 'utf-8', - 'accept-origin' => '*' - ], - 'providers' => ['context', 'adapt'] -]); - -/** - * @var \equal\php\Context $context - * @var \equal\data\DataAdapterProvider $dap - */ -list($context, $dap) = [$providers['context'], $providers['adapt']]; - -/** @var $adapter DataAdapter */ -$adapter = $dap->get('sql'); - -$import_config = getImportConfig( - QN_BASEDIR . '/packages/' . $params['package'] . '/init/import-config.json' -); - -$old_db_connection = createOldDbConnection( - $params['db_dbms'], $params['db_host'], $params['db_port'], $params['db_name'], - $params['db_user'], $params['db_password'], $params['db_charset'], $params['db_collation'] -); - -$new_db_connection = createNewDbConnection(); - -$limit = 500; - -foreach($import_config as $config) { - if(isset($params['entity']) && $params['entity'] !== $config['entity']) { - continue; - } - - $new_table_name = str_replace('\\', '_', $config['entity']); - - $offset = 0; - $remaining_data = true; - while($remaining_data) { - $res = $old_db_connection->getRecords( - $config['old_table']['name'], - array_keys($config['old_table']['fields']), - null, - $config['old_table']['conditions'] ?? null, - $config['old_table']['id_field'] ?? 'id', - [], - $offset, - $limit - ); - - if($res->num_rows < $limit) { - $remaining_data = false; - } - - $items = []; - while ($row = $old_db_connection->fetchArray($res)) { - $old_item = castOldDbRow($row, $config['old_table']['fields']); - $items[] = createNewItemFromOld($config, $old_db_connection, $old_item); - } - - if(!empty($items)) { - foreach($items as &$item) { - $item['created'] = $adapter->adaptOut($item['created'], 'datetime'); - $item['modified'] = $adapter->adaptOut($item['modified'], 'datetime'); - } - - $new_db_connection->addRecords( - $new_table_name, - array_keys($items[0]), - $items - ); - } - - $offset += $limit; - } -} - -$old_db_connection->disconnect(); -$new_db_connection->disconnect(); - -$context->httpResponse() - ->body(['success' => true]) - ->send(); - -function getImportConfig($config_file_path): array { +$getImportConfig = function($config_file_path): array { $config_file_content = file_get_contents($config_file_path); if(!$config_file_content) { throw new Exception('Missing import config file ' . $config_file_path, QN_ERROR_INVALID_CONFIG); @@ -161,9 +24,9 @@ function getImportConfig($config_file_path): array { } return $import_config; -} +}; -function createOldDbConnection(string $dbms, string $host, int $port, string $name, string $user, string $password, string $charset, string $collation) { +$createOldDbConnection = function(string $dbms, string $host, int $port, string $name, string $user, string $password, string $charset, string $collation) { $db_manipulator_map = [ 'MARIADB' => DBManipulatorMySQL::class, 'MYSQL' => DBManipulatorMySQL::class, @@ -186,9 +49,9 @@ function createOldDbConnection(string $dbms, string $host, int $port, string $na } return $db_connection; -} +}; -function createNewDbConnection() { +$createNewDbConnection = function() { $db_connection = DBConnection::getInstance( constant('DB_HOST'), constant('DB_PORT'), constant('DB_NAME'), constant('DB_USER'), constant('DB_PASSWORD'), @@ -201,9 +64,9 @@ function createNewDbConnection() { } return $db_connection; -} +}; -function castOldDbRow(array $old_item, array $old_table_fields_config): array { +$castOldDbRow = function(array $old_item, array $old_table_fields_config): array { $old_item_casted = []; foreach($old_item as $key => $column_value) { switch($old_table_fields_config[$key]) { @@ -226,9 +89,9 @@ function castOldDbRow(array $old_item, array $old_table_fields_config): array { } return $old_item_casted; -} +}; -function createNewItemFromOld(array $config, DBManipulator $old_db_connection, array $old_item): array { +$createNewItemFromOld = function(array $config, DBManipulator $old_db_connection, array $old_item): array { $item = [ 'creator' => QN_ROOT_USER_ID, 'created' => time(), @@ -333,4 +196,142 @@ function createNewItemFromOld(array $config, DBManipulator $old_db_connection, a } return $item; +}; + +list($params, $providers) = eQual::announce([ + 'description' => 'Import data from a database to eQual database for a given package.', + 'help' => 'Needs a configuration file init/import-config.json in the folder of the concerned package.', + 'params' => [ + 'db_dbms' => [ + 'type' => 'string', + 'default' => 'MYSQL' + ], + + 'db_host' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_port' => [ + 'type' => 'integer', + 'required' => true + ], + + 'db_user' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_password' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_name' => [ + 'type' => 'string', + 'required' => true + ], + + 'db_charset' => [ + 'type' => 'string', + 'default' => 'utf8mb4' + ], + + 'db_collation' => [ + 'type' => 'string', + 'default' => 'utf8mb4_unicode_ci' + ], + + 'package' => [ + 'type' => 'string', + 'required' => true + ], + + 'entity' => [ + 'type' => 'string' + ] + ], + 'response' => [ + 'content-type' => 'application/json', + 'charset' => 'utf-8', + 'accept-origin' => '*' + ], + 'providers' => ['context', 'adapt'] +]); + +/** + * @var \equal\php\Context $context + * @var \equal\data\DataAdapterProvider $dap + */ +list($context, $dap) = [$providers['context'], $providers['adapt']]; + +/** @var $adapter DataAdapter */ +$adapter = $dap->get('sql'); + +$import_config = $getImportConfig( + QN_BASEDIR . '/packages/' . $params['package'] . '/init/import-config.json' +); + +$old_db_connection = $createOldDbConnection( + $params['db_dbms'], $params['db_host'], $params['db_port'], $params['db_name'], + $params['db_user'], $params['db_password'], $params['db_charset'], $params['db_collation'] +); + +$new_db_connection = $createNewDbConnection(); + +$limit = 500; + +foreach($import_config as $config) { + if(isset($params['entity']) && $params['entity'] !== $config['entity']) { + continue; + } + + $new_table_name = str_replace('\\', '_', $config['entity']); + + $offset = 0; + $remaining_data = true; + while($remaining_data) { + $res = $old_db_connection->getRecords( + $config['old_table']['name'], + array_keys($config['old_table']['fields']), + null, + $config['old_table']['conditions'] ?? null, + $config['old_table']['id_field'] ?? 'id', + [], + $offset, + $limit + ); + + if($res->num_rows < $limit) { + $remaining_data = false; + } + + $items = []; + while ($row = $old_db_connection->fetchArray($res)) { + $old_item = $castOldDbRow($row, $config['old_table']['fields']); + $items[] = $createNewItemFromOld($config, $old_db_connection, $old_item); + } + + if(!empty($items)) { + foreach($items as &$item) { + $item['created'] = $adapter->adaptOut($item['created'], 'datetime'); + $item['modified'] = $adapter->adaptOut($item['modified'], 'datetime'); + } + + $new_db_connection->addRecords( + $new_table_name, + array_keys($items[0]), + $items + ); + } + + $offset += $limit; + } } + +$old_db_connection->disconnect(); +$new_db_connection->disconnect(); + +$context->httpResponse() + ->body(['success' => true]) + ->send();