From bcdedb53e8d0b3e5040f7e10d23e0eae64cd6228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Mon, 17 Jun 2024 09:54:36 +0200 Subject: [PATCH 01/10] Add datatype support getter to config --- src/Config/BaseConfig.php | 9 +++++++++ tests/Config/BaseConfigTest.php | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/Config/BaseConfig.php b/src/Config/BaseConfig.php index 27a1d56..6c68c79 100644 --- a/src/Config/BaseConfig.php +++ b/src/Config/BaseConfig.php @@ -289,4 +289,13 @@ public function getEnvKbcUrl(): string } return (string) $env; } + + public function getDataTypeSupport(): string + { + $env = getenv('KBC_DATA_TYPE_SUPPORT'); + if (!$env) { + throw new InvalidConfigurationException('The variable "KBC_DATA_TYPE_SUPPORT" is not allowed.'); + } + return $this->getStringValue(['storage', 'output', 'data_type_support'], (string) $env); + } } diff --git a/tests/Config/BaseConfigTest.php b/tests/Config/BaseConfigTest.php index 269064d..2d7262c 100644 --- a/tests/Config/BaseConfigTest.php +++ b/tests/Config/BaseConfigTest.php @@ -155,6 +155,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition return $nodeDefinition; } }; + putenv('KBC_DATA_TYPE_SUPPORT=authoritative'); $config = new BaseConfig([ 'parameters' => [ 'ipsum' => [ @@ -177,6 +178,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition 'tables' => [], ], 'output' => [ + 'data_type_support' => 'hint', 'files' => [], ], ], @@ -227,6 +229,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition 'tables' => [], ], 'output' => [ + 'data_type_support' => 'hint', 'files' => [], ], ], @@ -236,6 +239,10 @@ protected function getParametersDefinition(): ArrayNodeDefinition 'value', $config->getValue(['parameters', 'ipsum', 'dolor']), ); + $this->assertEquals( + 'hint', + $config->getDataTypeSupport(), + ); } /** @@ -313,6 +320,17 @@ public function testEnvGetters(array $envs): void Assert::assertEquals($envs['KBC_URL'], $config->getEnvKbcUrl()); } + if (!isset($envs['KBC_DATA_TYPE_SUPPORT'])) { + try { + $config->getDataTypeSupport(); + $this->fail('Should be fail.'); + } catch (InvalidConfigurationException $e) { + Assert::assertEquals('The variable "KBC_DATA_TYPE_SUPPORT" is not allowed.', $e->getMessage()); + } + } else { + Assert::assertEquals($envs['KBC_DATA_TYPE_SUPPORT'], $config->getDataTypeSupport()); + } + foreach ($envs as $env => $value) { putenv(sprintf('%s', $env)); } @@ -348,6 +366,7 @@ public function envGettersDataProvider(): Generator 'KBC_TOKENDESC' => 'tokenDesc', 'KBC_TOKEN' => 'token', 'KBC_URL' => 'url', + 'KBC_DATA_TYPE_SUPPORT' => 'authoritative', ], ]; } From ce1c7a7c5c543b25f48f89022af090ac804178fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Mon, 17 Jun 2024 09:55:16 +0200 Subject: [PATCH 02/10] Add delete_where_* properties to manifest options --- .../Options/OutTable/ManifestOptions.php | 39 +++++++++++++++++++ .../Options/OutTableManifestOptionsTest.php | 6 +++ 2 files changed, 45 insertions(+) diff --git a/src/Manifest/ManifestManager/Options/OutTable/ManifestOptions.php b/src/Manifest/ManifestManager/Options/OutTable/ManifestOptions.php index 3b119a8..820f18e 100644 --- a/src/Manifest/ManifestManager/Options/OutTable/ManifestOptions.php +++ b/src/Manifest/ManifestManager/Options/OutTable/ManifestOptions.php @@ -26,6 +26,12 @@ class ManifestOptions private ?string $enclosure = null; + private ?string $deleteWhereColumn = null; + + private ?array $deleteWhereValues = null; + + private ?string $deleteWhereOperator = null; + /** @var ManifestOptionsSchema[] */ private ?array $schema = null; @@ -342,4 +348,37 @@ private function validateMetadata($metadata): void } } } + + public function getDeleteWhereColumn(): ?string + { + return $this->deleteWhereColumn; + } + + public function getDeleteWhereValues(): ?array + { + return $this->deleteWhereValues; + } + + public function getDeleteWhereOperator(): ?string + { + return $this->deleteWhereOperator; + } + + public function setDeleteWhereColumn(?string $deleteWhereColumn): ManifestOptions + { + $this->deleteWhereColumn = $deleteWhereColumn; + return $this; + } + + public function setDeleteWhereValues(?array $deleteWhereValues): ManifestOptions + { + $this->deleteWhereValues = $deleteWhereValues; + return $this; + } + + public function setDeleteWhereOperator(?string $deleteWhereOperator): ManifestOptions + { + $this->deleteWhereOperator = $deleteWhereOperator; + return $this; + } } diff --git a/tests/Manifest/ManifestManager/Options/OutTableManifestOptionsTest.php b/tests/Manifest/ManifestManager/Options/OutTableManifestOptionsTest.php index 95c4245..c314025 100644 --- a/tests/Manifest/ManifestManager/Options/OutTableManifestOptionsTest.php +++ b/tests/Manifest/ManifestManager/Options/OutTableManifestOptionsTest.php @@ -51,6 +51,9 @@ public function provideOptions(): array [ 'destination' => 'my.table', 'manifest_type' => ManifestOptions::MANIFEST_TYPE_OUTPUT, + 'delete_where_column' => 'column1', + 'delete_where_values' => ['value1'], + 'delete_where_operator' => 'eq', 'delimiter' => '|', 'enclosure' => '_', 'incremental' => true, @@ -78,6 +81,9 @@ public function provideOptions(): array ], (new ManifestOptions()) ->setManifestType(ManifestOptions::MANIFEST_TYPE_OUTPUT) + ->setDeleteWhereColumn('column1') + ->setDeleteWhereValues(['value1']) + ->setDeleteWhereOperator('eq') ->setEnclosure('_') ->setDelimiter('|') ->setColumns(['id', 'number', 'other_column']) From 719029f428bc16767659d627e26ae9792dce4e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Mon, 17 Jun 2024 11:44:49 +0200 Subject: [PATCH 03/10] Fix reset env var in tests --- tests/Config/BaseConfigTest.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/Config/BaseConfigTest.php b/tests/Config/BaseConfigTest.php index 2d7262c..48b0203 100644 --- a/tests/Config/BaseConfigTest.php +++ b/tests/Config/BaseConfigTest.php @@ -251,7 +251,11 @@ protected function getParametersDefinition(): ArrayNodeDefinition public function testEnvGetters(array $envs): void { foreach ($envs as $env => $value) { - putenv(sprintf('%s=%s', $env, $value)); + if ($value === null) { + putenv($env); + } else { + putenv(sprintf('%s=%s', $env, $value)); + } } $config = new BaseConfig([], new BaseConfigDefinition()); @@ -348,6 +352,7 @@ public function envGettersDataProvider(): Generator 'KBC_COMPONENTID' => 'componentId', 'KBC_BRANCHID' => 'brancId', 'KBC_STAGING_FILE_PROVIDER' => 'staging_file_provider', + 'KBC_DATA_TYPE_SUPPORT' => null, ], ]; From 08013170d4e570c698bc0b098b1910a67c2ef5bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Tue, 18 Jun 2024 10:09:16 +0200 Subject: [PATCH 04/10] Set default data type support to none --- src/Config/BaseConfig.php | 8 ++++---- tests/Config/BaseConfigTest.php | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Config/BaseConfig.php b/src/Config/BaseConfig.php index 6c68c79..ca1a9c6 100644 --- a/src/Config/BaseConfig.php +++ b/src/Config/BaseConfig.php @@ -292,10 +292,10 @@ public function getEnvKbcUrl(): string public function getDataTypeSupport(): string { - $env = getenv('KBC_DATA_TYPE_SUPPORT'); - if (!$env) { - throw new InvalidConfigurationException('The variable "KBC_DATA_TYPE_SUPPORT" is not allowed.'); + $dataTypeSupport = getenv('KBC_DATA_TYPE_SUPPORT'); + if (!$dataTypeSupport) { + $dataTypeSupport = 'none'; } - return $this->getStringValue(['storage', 'output', 'data_type_support'], (string) $env); + return $this->getStringValue(['storage', 'output', 'data_type_support'], (string) $dataTypeSupport); } } diff --git a/tests/Config/BaseConfigTest.php b/tests/Config/BaseConfigTest.php index 48b0203..cd59d2e 100644 --- a/tests/Config/BaseConfigTest.php +++ b/tests/Config/BaseConfigTest.php @@ -325,12 +325,7 @@ public function testEnvGetters(array $envs): void } if (!isset($envs['KBC_DATA_TYPE_SUPPORT'])) { - try { - $config->getDataTypeSupport(); - $this->fail('Should be fail.'); - } catch (InvalidConfigurationException $e) { - Assert::assertEquals('The variable "KBC_DATA_TYPE_SUPPORT" is not allowed.', $e->getMessage()); - } + Assert::assertEquals('none', $config->getDataTypeSupport()); } else { Assert::assertEquals($envs['KBC_DATA_TYPE_SUPPORT'], $config->getDataTypeSupport()); } From 1058b1fd5294b9ec4eb292833ffe11d2f28d1b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Tue, 18 Jun 2024 10:09:51 +0200 Subject: [PATCH 05/10] fix typo --- tests/Config/BaseConfigTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Config/BaseConfigTest.php b/tests/Config/BaseConfigTest.php index cd59d2e..0e733a7 100644 --- a/tests/Config/BaseConfigTest.php +++ b/tests/Config/BaseConfigTest.php @@ -345,7 +345,7 @@ public function envGettersDataProvider(): Generator 'KBC_CONFIGID' => 'configId', 'KBC_CONFIGROWID' => 'configRowId', 'KBC_COMPONENTID' => 'componentId', - 'KBC_BRANCHID' => 'brancId', + 'KBC_BRANCHID' => 'branchId', 'KBC_STAGING_FILE_PROVIDER' => 'staging_file_provider', 'KBC_DATA_TYPE_SUPPORT' => null, ], @@ -359,7 +359,7 @@ public function envGettersDataProvider(): Generator 'KBC_CONFIGID' => 'configId', 'KBC_CONFIGROWID' => 'configRowId', 'KBC_COMPONENTID' => 'componentId', - 'KBC_BRANCHID' => 'brancId', + 'KBC_BRANCHID' => 'branchId', 'KBC_STAGING_FILE_PROVIDER' => 'staging_file_provider', 'KBC_PROJECTNAME' => 'projectName', 'KBC_TOKENID' => 'tokenId', From 1cad7b917afad0b28195edb4a6a5e1c77fed2e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Tue, 18 Jun 2024 10:17:21 +0200 Subject: [PATCH 06/10] Always return none if KBC_DATA_TYPE_SUPPORT is not set --- src/Config/BaseConfig.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Config/BaseConfig.php b/src/Config/BaseConfig.php index ca1a9c6..9e703e6 100644 --- a/src/Config/BaseConfig.php +++ b/src/Config/BaseConfig.php @@ -292,10 +292,10 @@ public function getEnvKbcUrl(): string public function getDataTypeSupport(): string { - $dataTypeSupport = getenv('KBC_DATA_TYPE_SUPPORT'); - if (!$dataTypeSupport) { - $dataTypeSupport = 'none'; + $env = getenv('KBC_DATA_TYPE_SUPPORT'); + if (!$env) { + return 'none'; } - return $this->getStringValue(['storage', 'output', 'data_type_support'], (string) $dataTypeSupport); + return $this->getStringValue(['storage', 'output', 'data_type_support'], (string) $env); } } From 536678c2cd7c129ad8e9caffa94d364b197787bd Mon Sep 17 00:00:00 2001 From: zajca Date: Tue, 18 Jun 2024 12:59:19 +0200 Subject: [PATCH 07/10] php 8.1+ DatatypeSupport enum --- .github/workflows/push.yml | 5 +---- composer.json | 2 +- src/Config/BaseConfig.php | 6 +++--- src/Config/DatatypeSupport.php | 17 +++++++++++++++++ tests/Config/BaseConfigTest.php | 7 ++++--- 5 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 src/Config/DatatypeSupport.php diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8170f99..d3d90b8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,11 +15,8 @@ jobs: strategy: fail-fast: false matrix: - phpVersion: [7.4, 8.0, 8.1] + phpVersion: [8.1, 8.2, 8.3] symfonyVersion: ["5.4.*", "6.1.*"] - exclude: - - phpVersion: 7.4 - symfonyVersion: "6.1.*" steps: - diff --git a/composer.json b/composer.json index f395685..44b084e 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ } }, "require": { - "php": "^7.4|^8.0", + "php": ">=8.1", "ext-json": "*", "keboola/common-exceptions": "^1.2", "monolog/monolog": "^2.3", diff --git a/src/Config/BaseConfig.php b/src/Config/BaseConfig.php index 9e703e6..5b088c6 100644 --- a/src/Config/BaseConfig.php +++ b/src/Config/BaseConfig.php @@ -290,12 +290,12 @@ public function getEnvKbcUrl(): string return (string) $env; } - public function getDataTypeSupport(): string + public function getDataTypeSupport(): DatatypeSupport { $env = getenv('KBC_DATA_TYPE_SUPPORT'); if (!$env) { - return 'none'; + return DatatypeSupport::NONE; } - return $this->getStringValue(['storage', 'output', 'data_type_support'], (string) $env); + return DatatypeSupport::from($this->getStringValue(['storage', 'output', 'data_type_support'], (string) $env)); } } diff --git a/src/Config/DatatypeSupport.php b/src/Config/DatatypeSupport.php new file mode 100644 index 0000000..451ec24 --- /dev/null +++ b/src/Config/DatatypeSupport.php @@ -0,0 +1,17 @@ +getValue(['parameters', 'ipsum', 'dolor']), ); $this->assertEquals( - 'hint', + DatatypeSupport::HINT, $config->getDataTypeSupport(), ); } @@ -325,9 +326,9 @@ public function testEnvGetters(array $envs): void } if (!isset($envs['KBC_DATA_TYPE_SUPPORT'])) { - Assert::assertEquals('none', $config->getDataTypeSupport()); + Assert::assertEquals(DatatypeSupport::NONE, $config->getDataTypeSupport()); } else { - Assert::assertEquals($envs['KBC_DATA_TYPE_SUPPORT'], $config->getDataTypeSupport()); + Assert::assertEquals(DatatypeSupport::from($envs['KBC_DATA_TYPE_SUPPORT']), $config->getDataTypeSupport()); } foreach ($envs as $env => $value) { From 42ee7033487cd93361a8bca5af87db9386e7615d Mon Sep 17 00:00:00 2001 From: zajca Date: Tue, 18 Jun 2024 13:38:39 +0200 Subject: [PATCH 08/10] getDataTypeSupport handle invalid input --- src/Config/BaseConfig.php | 10 +++++++-- tests/Config/BaseConfigTest.php | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/Config/BaseConfig.php b/src/Config/BaseConfig.php index 5b088c6..6c04858 100644 --- a/src/Config/BaseConfig.php +++ b/src/Config/BaseConfig.php @@ -290,12 +290,18 @@ public function getEnvKbcUrl(): string return (string) $env; } - public function getDataTypeSupport(): DatatypeSupport + public function getDataTypeSupport(): ?DatatypeSupport { $env = getenv('KBC_DATA_TYPE_SUPPORT'); if (!$env) { return DatatypeSupport::NONE; } - return DatatypeSupport::from($this->getStringValue(['storage', 'output', 'data_type_support'], (string) $env)); + $datatypeSupport = DatatypeSupport::tryFrom($this->getStringValue([ + 'storage', + 'output', + 'data_type_support', + ], (string) $env)); + + return $datatypeSupport ?? DatatypeSupport::from((string) $env); } } diff --git a/tests/Config/BaseConfigTest.php b/tests/Config/BaseConfigTest.php index 0081edb..51e479a 100644 --- a/tests/Config/BaseConfigTest.php +++ b/tests/Config/BaseConfigTest.php @@ -246,6 +246,42 @@ protected function getParametersDefinition(): ArrayNodeDefinition ); } + public function testGetDataTypeSupportUserInvalidInput(): void + { + putenv('KBC_DATA_TYPE_SUPPORT=authoritative'); + $config = new BaseConfig([ + 'storage' => [ + 'output' => [ + 'data_type_support' => 'I\'m invalid', + 'files' => [], + ], + ], + ], new BaseConfigDefinition); + + $this->assertEquals( + DatatypeSupport::AUTHORITATIVE, // from env + $config->getDataTypeSupport(), + ); + } + + public function testGetDataTypeSupportNoEnvUserInvalidInput(): void + { + putenv('KBC_DATA_TYPE_SUPPORT='); + $config = new BaseConfig([ + 'storage' => [ + 'output' => [ + 'data_type_support' => 'I\'m invalid', + 'files' => [], + ], + ], + ], new BaseConfigDefinition); + + $this->assertEquals( + DatatypeSupport::NONE, // default when env not setZ + $config->getDataTypeSupport(), + ); + } + /** * @dataProvider envGettersDataProvider */ From e222064c2550a22a0fec824b36be1be40e7c1e42 Mon Sep 17 00:00:00 2001 From: zajca Date: Tue, 18 Jun 2024 13:44:29 +0200 Subject: [PATCH 09/10] fix getDataTypeSupport typehint --- src/Config/BaseConfig.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config/BaseConfig.php b/src/Config/BaseConfig.php index 6c04858..c30ca55 100644 --- a/src/Config/BaseConfig.php +++ b/src/Config/BaseConfig.php @@ -290,7 +290,7 @@ public function getEnvKbcUrl(): string return (string) $env; } - public function getDataTypeSupport(): ?DatatypeSupport + public function getDataTypeSupport(): DatatypeSupport { $env = getenv('KBC_DATA_TYPE_SUPPORT'); if (!$env) { From 4d204418f8ada62d3f51118f32f344f1196decbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=BDborn=C3=BD=20Adam?= Date: Tue, 18 Jun 2024 15:49:11 +0200 Subject: [PATCH 10/10] Fix typos --- src/Config/DatatypeSupport.php | 2 +- tests/Config/BaseConfigTest.php | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Config/DatatypeSupport.php b/src/Config/DatatypeSupport.php index 451ec24..9858aaf 100644 --- a/src/Config/DatatypeSupport.php +++ b/src/Config/DatatypeSupport.php @@ -7,7 +7,7 @@ enum DatatypeSupport: string { case AUTHORITATIVE = 'authoritative'; - case HINT = 'hint'; + case HINTS = 'hints'; case NONE = 'none'; public function usingLegacyManifest(): bool diff --git a/tests/Config/BaseConfigTest.php b/tests/Config/BaseConfigTest.php index 51e479a..77cf779 100644 --- a/tests/Config/BaseConfigTest.php +++ b/tests/Config/BaseConfigTest.php @@ -179,7 +179,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition 'tables' => [], ], 'output' => [ - 'data_type_support' => 'hint', + 'data_type_support' => DatatypeSupport::HINTS->value, 'files' => [], ], ], @@ -230,7 +230,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition 'tables' => [], ], 'output' => [ - 'data_type_support' => 'hint', + 'data_type_support' => DatatypeSupport::HINTS->value, 'files' => [], ], ], @@ -241,7 +241,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition $config->getValue(['parameters', 'ipsum', 'dolor']), ); $this->assertEquals( - DatatypeSupport::HINT, + DatatypeSupport::HINTS, $config->getDataTypeSupport(), ); } @@ -277,7 +277,7 @@ public function testGetDataTypeSupportNoEnvUserInvalidInput(): void ], new BaseConfigDefinition); $this->assertEquals( - DatatypeSupport::NONE, // default when env not setZ + DatatypeSupport::NONE, // default when env not set $config->getDataTypeSupport(), ); } @@ -403,7 +403,7 @@ public function envGettersDataProvider(): Generator 'KBC_TOKENDESC' => 'tokenDesc', 'KBC_TOKEN' => 'token', 'KBC_URL' => 'url', - 'KBC_DATA_TYPE_SUPPORT' => 'authoritative', + 'KBC_DATA_TYPE_SUPPORT' => DatatypeSupport::AUTHORITATIVE->value, ], ]; }