New native datatypes fix ups

.github/workflows/push.yml

@@ -15,11 +15,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        phpVersion: [7.4, 8.0, 8.1]
+        phpVersion: [8.1, 8.2, 8.3]
         symfonyVersion: ["5.4.*", "6.1.*"]
-        exclude:
-          - phpVersion: 7.4
-            symfonyVersion: "6.1.*"
 
     steps:
       -

composer.json

@@ -26,7 +26,7 @@
         }
     },
     "require": {
-        "php": "^7.4|^8.0",
+        "php": ">=8.1",
         "ext-json": "*",
         "keboola/common-exceptions": "^1.2",
         "monolog/monolog": "^2.3",

src/Config/BaseConfig.php

@@ -289,4 +289,19 @@ public function getEnvKbcUrl(): string
         }
         return (string) $env;
     }
+
+    public function getDataTypeSupport(): DatatypeSupport
+    {
+        $env = getenv('KBC_DATA_TYPE_SUPPORT');
+        if (!$env) {
+            return DatatypeSupport::NONE;
+        }
+        $datatypeSupport = DatatypeSupport::tryFrom($this->getStringValue([
+            'storage',
+            'output',
+            'data_type_support',
+        ], (string) $env));
+
+        return $datatypeSupport ?? DatatypeSupport::from((string) $env);
+    }
 }

src/Config/DatatypeSupport.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\Component\Config;
+
+enum DatatypeSupport: string
+{
+    case AUTHORITATIVE = 'authoritative';
+    case HINTS = 'hints';
+    case NONE = 'none';
+
+    public function usingLegacyManifest(): bool
+    {
+        return $this === self::NONE;
+    }
+}

src/Manifest/ManifestManager/Options/OutTable/ManifestOptions.php

@@ -26,6 +26,12 @@ class ManifestOptions
 
     private ?string $enclosure = null;
 
+    private ?string $deleteWhereColumn = null;
+
+    private ?array $deleteWhereValues = null;
+
+    private ?string $deleteWhereOperator = null;
+
     /** @var ManifestOptionsSchema[]  */
     private ?array $schema = null;
 
@@ -342,4 +348,37 @@ private function validateMetadata($metadata): void
             }
         }
     }
+
+    public function getDeleteWhereColumn(): ?string
+    {
+        return $this->deleteWhereColumn;
+    }
+
+    public function getDeleteWhereValues(): ?array
+    {
+        return $this->deleteWhereValues;
+    }
+
+    public function getDeleteWhereOperator(): ?string
+    {
+        return $this->deleteWhereOperator;
+    }
+
+    public function setDeleteWhereColumn(?string $deleteWhereColumn): ManifestOptions
+    {
+        $this->deleteWhereColumn = $deleteWhereColumn;
+        return $this;
+    }
+
+    public function setDeleteWhereValues(?array $deleteWhereValues): ManifestOptions
+    {
+        $this->deleteWhereValues = $deleteWhereValues;
+        return $this;
+    }
+
+    public function setDeleteWhereOperator(?string $deleteWhereOperator): ManifestOptions
+    {
+        $this->deleteWhereOperator = $deleteWhereOperator;
+        return $this;
+    }
 }

tests/Config/BaseConfigTest.php

@@ -7,6 +7,7 @@
 use Generator;
 use Keboola\Component\Config\BaseConfig;
 use Keboola\Component\Config\BaseConfigDefinition;
+use Keboola\Component\Config\DatatypeSupport;
 use PHPUnit\Framework\Assert;
 use PHPUnit\Framework\TestCase;
 use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
@@ -155,6 +156,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition
                 return $nodeDefinition;
             }
         };
+        putenv('KBC_DATA_TYPE_SUPPORT=authoritative');
         $config = new BaseConfig([
             'parameters' => [
                 'ipsum' => [
@@ -177,6 +179,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition
                     'tables' => [],
                 ],
                 'output' => [
+                    'data_type_support' => DatatypeSupport::HINTS->value,
                     'files' => [],
                 ],
             ],
@@ -227,6 +230,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition
                     'tables' => [],
                 ],
                 'output' => [
+                    'data_type_support' => DatatypeSupport::HINTS->value,
                     'files' => [],
                 ],
             ],
@@ -236,6 +240,46 @@ protected function getParametersDefinition(): ArrayNodeDefinition
             'value',
             $config->getValue(['parameters', 'ipsum', 'dolor']),
         );
+        $this->assertEquals(
+            DatatypeSupport::HINTS,
+            $config->getDataTypeSupport(),
+        );
+    }
+
+    public function testGetDataTypeSupportUserInvalidInput(): void
+    {
+        putenv('KBC_DATA_TYPE_SUPPORT=authoritative');
+        $config = new BaseConfig([
+            'storage' => [
+                'output' => [
+                    'data_type_support' => 'I\'m invalid',
+                    'files' => [],
+                ],
+            ],
+        ], new BaseConfigDefinition);
+
+        $this->assertEquals(
+            DatatypeSupport::AUTHORITATIVE, // from env
+            $config->getDataTypeSupport(),
+        );
+    }
+
+    public function testGetDataTypeSupportNoEnvUserInvalidInput(): void
+    {
+        putenv('KBC_DATA_TYPE_SUPPORT=');
+        $config = new BaseConfig([
+            'storage' => [
+                'output' => [
+                    'data_type_support' => 'I\'m invalid',
+                    'files' => [],
+                ],
+            ],
+        ], new BaseConfigDefinition);
+
+        $this->assertEquals(
+            DatatypeSupport::NONE, // default when env not set
+            $config->getDataTypeSupport(),
+        );
     }
 
     /**
@@ -244,7 +288,11 @@ protected function getParametersDefinition(): ArrayNodeDefinition
     public function testEnvGetters(array $envs): void
     {
         foreach ($envs as $env => $value) {
-            putenv(sprintf('%s=%s', $env, $value));
+            if ($value === null) {
+                putenv($env);
+            } else {
+                putenv(sprintf('%s=%s', $env, $value));
+            }
         }
 
         $config = new BaseConfig([], new BaseConfigDefinition());
@@ -313,6 +361,12 @@ public function testEnvGetters(array $envs): void
             Assert::assertEquals($envs['KBC_URL'], $config->getEnvKbcUrl());
         }
 
+        if (!isset($envs['KBC_DATA_TYPE_SUPPORT'])) {
+            Assert::assertEquals(DatatypeSupport::NONE, $config->getDataTypeSupport());
+        } else {
+            Assert::assertEquals(DatatypeSupport::from($envs['KBC_DATA_TYPE_SUPPORT']), $config->getDataTypeSupport());
+        }
+
         foreach ($envs as $env => $value) {
             putenv(sprintf('%s', $env));
         }
@@ -328,8 +382,9 @@ public function envGettersDataProvider(): Generator
                 'KBC_CONFIGID' => 'configId',
                 'KBC_CONFIGROWID' => 'configRowId',
                 'KBC_COMPONENTID' => 'componentId',
-                'KBC_BRANCHID' => 'brancId',
+                'KBC_BRANCHID' => 'branchId',
                 'KBC_STAGING_FILE_PROVIDER' => 'staging_file_provider',
+                'KBC_DATA_TYPE_SUPPORT' => null,
             ],
         ];
 
@@ -341,13 +396,14 @@ public function envGettersDataProvider(): Generator
                 'KBC_CONFIGID' => 'configId',
                 'KBC_CONFIGROWID' => 'configRowId',
                 'KBC_COMPONENTID' => 'componentId',
-                'KBC_BRANCHID' => 'brancId',
+                'KBC_BRANCHID' => 'branchId',
                 'KBC_STAGING_FILE_PROVIDER' => 'staging_file_provider',
                 'KBC_PROJECTNAME' => 'projectName',
                 'KBC_TOKENID' => 'tokenId',
                 'KBC_TOKENDESC' => 'tokenDesc',
                 'KBC_TOKEN' => 'token',
                 'KBC_URL' => 'url',
+                'KBC_DATA_TYPE_SUPPORT' => DatatypeSupport::AUTHORITATIVE->value,
             ],
         ];
     }

tests/Manifest/ManifestManager/Options/OutTableManifestOptionsTest.php

@@ -51,6 +51,9 @@ public function provideOptions(): array
                 [
                     'destination' => 'my.table',
                     'manifest_type' => ManifestOptions::MANIFEST_TYPE_OUTPUT,
+                    'delete_where_column' => 'column1',
+                    'delete_where_values' => ['value1'],
+                    'delete_where_operator' => 'eq',
                     'delimiter' => '|',
                     'enclosure' => '_',
                     'incremental' => true,
@@ -78,6 +81,9 @@ public function provideOptions(): array
                 ],
                 (new ManifestOptions())
                     ->setManifestType(ManifestOptions::MANIFEST_TYPE_OUTPUT)
+                    ->setDeleteWhereColumn('column1')
+                    ->setDeleteWhereValues(['value1'])
+                    ->setDeleteWhereOperator('eq')
                     ->setEnclosure('_')
                     ->setDelimiter('|')
                     ->setColumns(['id', 'number', 'other_column'])