wip - functional tests

Dockerfile

@@ -35,6 +35,10 @@ RUN docker-php-ext-install pdo_mysql
 RUN docker-php-ext-configure intl \
     && docker-php-ext-install intl
 
+# Xdebug
+RUN pecl install xdebug \
+ && docker-php-ext-enable xdebug
+
 ## Composer - deps always cached unless changed
 # First copy only composer files
 COPY composer.* /code/

composer.json

@@ -27,13 +27,17 @@
   },
   "autoload-dev": {
     "psr-4": {
+      "Keboola\\DbWriter\\Traits\\": "tests/Traits/",
+      "Keboola\\DbWriter\\TestsFunctional\\": "tests/functional/",
       "Keboola\\DbWriter\\Tests\\": "tests/phpunit/"
     }
   },
   "scripts": {
-    "tests-phpunit": "phpunit",
+    "tests-phpunit": "phpunit --testsuite unit",
+    "tests-datadir": "phpunit --testsuite functional",
     "tests": [
-      "@tests-phpunit"
+      "@tests-phpunit",
+      "@tests-datadir"
     ],
     "phpcs": "phpcs -n --ignore=vendor,tests --extensions=php .",
     "phpcbf": "phpcbf -n --ignore=vendor --extensions=php .",
@@ -57,7 +61,8 @@
   },
   "require-dev": {
     "keboola/coding-standard": "^15.0",
+    "keboola/datadir-tests": "dev-ondra-update-packages",
     "phpstan/phpstan": "^1.10",
-    "phpunit/phpunit": "^10.4"
+    "phpunit/phpunit": "^9.6"
   }
 }

phpunit.xml.dist

@@ -9,7 +9,12 @@
          stopOnFailure="false"
          bootstrap="vendor/autoload.php">
 
-    <testsuite name="DB Writer Common Test Suite">
-        <directory>tests/phpunit</directory>
-    </testsuite>
+    <testsuites>
+        <testsuite name="DB Writer Common PHPunit">
+            <directory>tests/phpunit</directory>
+        </testsuite>
+        <testsuite name="DB Writer MySQL Datadir">
+            <directory>tests/functional</directory>
+        </testsuite>
+    </testsuites>
 </phpunit>

src/Application.php

@@ -38,8 +38,10 @@ protected function run(): void
 
         if (!$this->isRowConfiguration($parameters)) {
             $filteredTables = array_filter($parameters['tables'], fn($table) => $table['export']);
+            unset($parameters['tables']);
             foreach ($filteredTables as $filteredTable) {
                 $filteredTable = $this->validateTableItems($filteredTable);
+                $filteredTable = array_merge($parameters, $filteredTable);
                 $writer->write($this->createExportConfig($filteredTable));
             }
         } else {
@@ -102,7 +104,7 @@ protected function getSyncActions(): array
 
     protected function getConfigDefinitionClass(): string
     {
-        if ($this->isRowConfiguration($this->getRawConfig())) {
+        if ($this->isRowConfiguration($this->getRawConfig()['parameters'])) {
             $action = $this->getRawConfig()['action'] ?? 'run';
             if ($action === 'run') {
                 return ConfigRowDefinition::class;

src/Writer/Common.php

@@ -53,6 +53,7 @@ protected function createWriteAdapter(): WriteAdapter
         return new PdoWriteAdapter(
             $this->connection,
             new DefaultQueryBuilder(),
+            $this->logger,
         );
     }
 }

src/run.php → tests/Fixtures/run.php

@@ -6,9 +6,13 @@
 use Keboola\Component\Logger;
 use Keboola\DbWriter\Application;
 
-require __DIR__ . '/../vendor/autoload.php';
+require __DIR__ . '/../../vendor/autoload.php';
 
 $logger = new Logger();
+
+$app = new Application($logger);
+$app->execute();
+exit(0);
 try {
     $app = new Application($logger);
     $app->execute();

tests/functional/DatadirTest.php

@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\DbWriter\TestsFunctional;
+
+use Keboola\DatadirTests\DatadirTestCase;
+use Keboola\DbWriter\Traits\CloseSshTunnelsTrait;
+use Keboola\DbWriter\Traits\DropAllTablesTrait;
+use Keboola\DbWriterAdapter\PDO\PdoConnection;
+use RuntimeException;
+
+class DatadirTest extends DatadirTestCase
+{
+    use CloseSshTunnelsTrait;
+    use DropAllTablesTrait;
+
+    public PdoConnection $connection;
+
+    protected function getScript(): string
+    {
+        return $this->getTestFileDir() . '/../../tests/Fixtures/run.php';
+    }
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+        putenv('KBC_COMPONENT_RUN_MODE=run');
+
+        // Test dir, eg. "/code/tests/functional/full-load-ok"
+        $this->testProjectDir = $this->getTestFileDir() . '/' . $this->dataName();
+        $this->testTempDir = $this->temp->getTmpFolder();
+
+        $isSsl = str_starts_with((string) $this->dataName(), 'ssl-');
+        $this->connection = PdoTestConnection::createConnection();
+        $this->closeSshTunnels();
+        $this->dropAllTables($this->connection);
+
+        // Load setUp.php file - used to init database state
+        $setUpPhpFile = $this->testProjectDir . '/setUp.php';
+        if (file_exists($setUpPhpFile)) {
+            // Get callback from file and check it
+            $initCallback = require $setUpPhpFile;
+            if (!is_callable($initCallback)) {
+                throw new RuntimeException(sprintf('File "%s" must return callback!', $setUpPhpFile));
+            }
+
+            // Invoke callback
+            $initCallback($this);
+        }
+    }
+}

tests/functional/PdoTestConnection.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\DbWriter\TestsFunctional;
+
+use Keboola\Component\UserException;
+use Keboola\DbWriterAdapter\PDO\PdoConnection;
+use Psr\Log\NullLogger;
+
+class PdoTestConnection
+{
+    public static function getDbConfigArray(bool $ssl = false): array
+    {
+        $config = [
+            'host' => getenv('COMMON_DB_HOST', true) ?: (string) getenv('COMMON_DB_HOST'),
+            'port' => (string) getenv('COMMON_DB_PORT'),
+            'user' => (string) getenv('COMMON_DB_USER'),
+            '#password' => (string) getenv('COMMON_DB_PASSWORD'),
+            'database' => (string) getenv('COMMON_DB_DATABASE'),
+        ];
+
+        return $config;
+    }
+
+    /**
+     * @throws UserException
+     */
+    public static function createConnection(): PdoConnection
+    {
+        return new PdoConnection(
+            new NullLogger(),
+            sprintf(
+                'mysql:host=%s;port=%s;dbname=%s;charset=utf8',
+                self::getDbConfigArray()['host'],
+                self::getDbConfigArray()['port'],
+                self::getDbConfigArray()['database'],
+            ),
+            self::getDbConfigArray()['user'],
+            self::getDbConfigArray()['#password'],
+            [],
+        );
+    }
+}

tests/functional/get-tables-info/expected-code

@@ -0,0 +1 @@
+0

tests/functional/get-tables-info/expected-stdout

@@ -0,0 +1 @@
+{"status":"success","tables":{"simple":[{"Field":"id","Type":"int(11)","Null":"NO","Key":"PRI","Default":null,"Extra":"auto_increment"},{"Field":"name","Type":"varchar(255)","Null":"NO","Key":"","Default":null,"Extra":""},{"Field":"value","Type":"int(11)","Null":"NO","Key":"","Default":null,"Extra":""}]}}

tests/functional/get-tables-info/setUp.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+use Keboola\DbWriter\TestsFunctional\DatadirTest;
+
+return function (DatadirTest $test): void {
+    $sql = <<<SQL
+CREATE TABLE `simple` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `value` int(11) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+SQL;
+
+    $test->connection->exec($sql);
+};

tests/functional/get-tables-info/source/data/config.json

@@ -0,0 +1,15 @@
+{
+  "action": "getTablesInfo",
+  "parameters": {
+    "writer_class": "Common",
+    "data_dir": "/data",
+    "db": {
+      "driver": "mysql",
+      "host": "%env(string:COMMON_DB_HOST)%",
+      "port": "%env(string:COMMON_DB_PORT)%",
+      "database": "%env(string:COMMON_DB_DATABASE)%",
+      "user": "%env(string:COMMON_DB_USER)%",
+      "#password": "%env(string:COMMON_DB_PASSWORD)%"
+    }
+  }
+}

tests/functional/run-action/expected-code

@@ -0,0 +1 @@
+0

tests/functional/run-action/expected-stdout

@@ -0,0 +1,7 @@
+Creating PDO connection to "mysql:host=mysql;port=3306;dbname=testdb;charset=utf8".
+Table "encoding" created
+Data written to table "encoding".
+Temporary Table "%s" created
+Data written to table "%s".
+Table "simple" created
+Data upserted to table "simple".

tests/functional/run-action/source/data/config.json

@@ -0,0 +1,102 @@
+{
+  "parameters": {
+    "writer_class": "Common",
+    "data_dir": "/data",
+    "db": {
+      "driver": "mysql",
+      "host": "%env(string:COMMON_DB_HOST)%",
+      "port": "%env(string:COMMON_DB_PORT)%",
+      "database": "%env(string:COMMON_DB_DATABASE)%",
+      "user": "%env(string:COMMON_DB_USER)%",
+      "#password": "%env(string:COMMON_DB_PASSWORD)%"
+    },
+    "tables": [
+      {
+        "tableId": "encoding",
+        "dbName": "encoding",
+        "export": true,
+        "incremental": false,
+        "primaryKey": [],
+        "items": [
+          {
+            "name": "col1",
+            "dbName": "col1",
+            "type": "VARCHAR",
+            "size": 255,
+            "nullable": null,
+            "default": null
+          },
+          {
+            "name": "col2",
+            "dbName": "col2",
+            "type": "VARCHAR",
+            "size": 255,
+            "nullable": null,
+            "default": null
+          }
+        ]
+      },
+      {
+        "tableId": "simple",
+        "dbName": "simple",
+        "export": true,
+        "incremental": true,
+        "primaryKey": [
+          "id"
+        ],
+        "items": [
+          {
+            "name": "id",
+            "dbName": "id",
+            "type": "int",
+            "size": null,
+            "nullable": null,
+            "default": null
+          },
+          {
+            "name": "name",
+            "dbName": "name",
+            "type": "VARCHAR",
+            "size": 255,
+            "nullable": null,
+            "default": null
+          },
+          {
+            "name": "glasses",
+            "dbName": "glasses",
+            "type": "VARCHAR",
+            "size": 20,
+            "nullable": null,
+            "default": null
+          }
+        ]
+      }
+    ]
+  },
+  "storage": {
+    "input": {
+      "tables": [
+        {
+          "source": "encoding",
+          "destination": "encoding.csv",
+          "columns": [
+            "id",
+            "name",
+            "glasses",
+            "age"
+          ]
+        },
+        {
+          "source": "simple",
+          "destination": "simple.csv",
+          "columns": [
+            "id",
+            "name",
+            "glasses",
+            "age"
+          ]
+        }
+      ]
+    }
+  }
+}

tests/phpunit/Base.php

@@ -0,0 +1,24 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\DbWriter\Tests;
+
+use PHPUnit\Framework\TestCase;
+use Symfony\Component\Process\Process;
+
+class Base extends TestCase
+{
+    protected function setUp(): void
+    {
+        $this->closeSshTunnels();
+        parent::setUp();
+    }
+
+    protected function closeSshTunnels(): void
+    {
+        # Close SSH tunnel if created
+        $process = new Process(['sh', '-c', 'pgrep ssh | xargs -r kill']);
+        $process->mustRun();
+    }
+}

tests/phpunit/data/deprecated/in/tables/simple_increment.csv

@@ -0,0 +1,5 @@
+"id","name","glasses"
+"7","Jakub Matejka","no"
+"8","Ondrej Popelka","no"
+"9","Pavel Dolezal","yes"
+"10","Klara Koudelova","yes"

tests/phpunit/data/deprecated/in/tables/simple_increment.csv.manifest

@@ -0,0 +1,7 @@
+{
+    "columns": [
+        "id",
+        "name",
+        "glasses"
+    ]
+}