wip - functional tests

Dockerfile

@@ -17,7 +17,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git  \
     bzip2  \
     time  \
-    libzip-dev  \
+    libzip-dev \
+    ssh \
     openssl  \
     unzip  \
     procps \

composer.json

@@ -61,6 +61,7 @@
   },
   "require-dev": {
     "keboola/coding-standard": "^15.0",
+    "keboola/csv": "^4.0",
     "keboola/datadir-tests": "dev-ondra-update-packages",
     "phpstan/phpstan": "^1.10",
     "phpunit/phpunit": "^9.6"

tests/Fixtures/run.php

@@ -10,9 +10,6 @@
 
 $logger = new Logger();
 
-$app = new Application($logger);
-$app->execute();
-exit(0);
 try {
     $app = new Application($logger);
     $app->execute();

tests/functional/DatadirTest.php

@@ -4,7 +4,9 @@
 
 namespace Keboola\DbWriter\TestsFunctional;
 
+use Keboola\Csv\CsvWriter;
 use Keboola\DatadirTests\DatadirTestCase;
+use Keboola\DatadirTests\DatadirTestSpecificationInterface;
 use Keboola\DbWriter\Traits\CloseSshTunnelsTrait;
 use Keboola\DbWriter\Traits\DropAllTablesTrait;
 use Keboola\DbWriterAdapter\PDO\PdoConnection;
@@ -49,4 +51,38 @@ protected function setUp(): void
             $initCallback($this);
         }
     }
+
+    /**
+     * @dataProvider provideDatadirSpecifications
+     */
+    public function testDatadir(DatadirTestSpecificationInterface $specification): void
+    {
+        $tempDatadir = $this->getTempDatadir($specification);
+
+        $process = $this->runScript($tempDatadir->getTmpFolder());
+
+        $this->exportTablesData($tempDatadir->getTmpFolder());
+
+        $this->assertMatchesSpecification($specification, $process, $tempDatadir->getTmpFolder());
+    }
+
+    protected function exportTablesData($testTempDir): void
+    {
+        $sqlTables = <<<SQL
+SELECT `table_schema`,`table_name`
+FROM information_schema.tables 
+WHERE TABLE_SCHEMA NOT IN ("performance_schema", "mysql", "information_schema", "sys");
+SQL;
+
+        $tables = $this->connection->fetchAll($sqlTables, 3);
+
+        foreach ($tables as $table) {
+            $sql = sprintf('SELECT * FROM `%s`.`%s`', $table['table_schema'], $table['table_name']);
+            $data = $this->connection->fetchAll($sql, 3);
+            $csv = new CsvWriter(sprintf('%s/out/tables/%s.csv', $testTempDir, $table['table_name']));
+            foreach ($data as $item) {
+                $csv->writeRow($item);
+            }
+        }
+    }
 }

tests/functional/check-hostname/expected-code

@@ -0,0 +1 @@
+1

tests/functional/check-hostname/expected-stderr

@@ -0,0 +1 @@
+Hostname "mysql" with port "3306" is not approved.

tests/functional/check-hostname/setUp.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+use Keboola\DbWriter\TestsFunctional\DatadirTest;
+
+return function (DatadirTest $test): void {
+    $sql = <<<SQL
+CREATE TABLE `simple` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `value` int(11) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+SQL;
+
+    $test->connection->exec($sql);
+};

tests/functional/check-hostname/source/data/config.json

@@ -0,0 +1,23 @@
+{
+  "action": "getTablesInfo",
+  "parameters": {
+    "writer_class": "Common",
+    "data_dir": "/data",
+    "db": {
+      "driver": "mysql",
+      "host": "%env(string:COMMON_DB_HOST)%",
+      "port": "%env(string:COMMON_DB_PORT)%",
+      "database": "%env(string:COMMON_DB_DATABASE)%",
+      "user": "%env(string:COMMON_DB_USER)%",
+      "#password": "%env(string:COMMON_DB_PASSWORD)%"
+    }
+  },
+  "image_parameters": {
+    "approvedHostnames": [
+      {
+        "host": "unknownHostname",
+        "port": "%env(string:COMMON_DB_PORT)%"
+      }
+    ]
+  }
+}

tests/functional/error-check-hostname-empty-array/expected-code

@@ -0,0 +1 @@
+1

tests/functional/error-check-hostname-empty-array/expected-stderr

@@ -0,0 +1 @@
+Hostname "mysql" with port "3306" is not approved.

tests/functional/error-check-hostname-empty-array/setUp.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+use Keboola\DbWriter\TestsFunctional\DatadirTest;
+
+return function (DatadirTest $test): void {
+    $sql = <<<SQL
+CREATE TABLE `simple` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `value` int(11) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+SQL;
+
+    $test->connection->exec($sql);
+};

tests/functional/error-check-hostname-empty-array/source/data/config.json

@@ -0,0 +1,18 @@
+{
+  "action": "getTablesInfo",
+  "parameters": {
+    "writer_class": "Common",
+    "data_dir": "/data",
+    "db": {
+      "driver": "mysql",
+      "host": "%env(string:COMMON_DB_HOST)%",
+      "port": "%env(string:COMMON_DB_PORT)%",
+      "database": "%env(string:COMMON_DB_DATABASE)%",
+      "user": "%env(string:COMMON_DB_USER)%",
+      "#password": "%env(string:COMMON_DB_PASSWORD)%"
+    }
+  },
+  "image_parameters": {
+    "approvedHostnames": []
+  }
+}

tests/functional/error-check-hostname/expected-code

@@ -0,0 +1 @@
+0

tests/functional/error-check-hostname/expected-stdout

@@ -0,0 +1 @@
+{"status":"success","tables":{"simple":[{"Field":"id","Type":"int(11)","Null":"NO","Key":"PRI","Default":null,"Extra":"auto_increment"},{"Field":"name","Type":"varchar(255)","Null":"NO","Key":"","Default":null,"Extra":""},{"Field":"value","Type":"int(11)","Null":"NO","Key":"","Default":null,"Extra":""}]}}

tests/functional/error-check-hostname/setUp.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+use Keboola\DbWriter\TestsFunctional\DatadirTest;
+
+return function (DatadirTest $test): void {
+    $sql = <<<SQL
+CREATE TABLE `simple` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `value` int(11) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+SQL;
+
+    $test->connection->exec($sql);
+};

tests/functional/error-check-hostname/source/data/config.json

@@ -0,0 +1,23 @@
+{
+  "action": "getTablesInfo",
+  "parameters": {
+    "writer_class": "Common",
+    "data_dir": "/data",
+    "db": {
+      "driver": "mysql",
+      "host": "%env(string:COMMON_DB_HOST)%",
+      "port": "%env(string:COMMON_DB_PORT)%",
+      "database": "%env(string:COMMON_DB_DATABASE)%",
+      "user": "%env(string:COMMON_DB_USER)%",
+      "#password": "%env(string:COMMON_DB_PASSWORD)%"
+    }
+  },
+  "image_parameters": {
+    "approvedHostnames": [
+      {
+        "host": "%env(string:COMMON_DB_HOST)%",
+        "port": "%env(string:COMMON_DB_PORT)%"
+      }
+    ]
+  }
+}

tests/functional/error-invalid-input-mapping/expected-code

@@ -0,0 +1 @@
+1

tests/functional/error-invalid-input-mapping/expected-stderr

@@ -0,0 +1 @@
+Table "invalidtable" in storage input mapping cannot be found.

tests/functional/error-invalid-input-mapping/expected-stdout

@@ -0,0 +1 @@
+Creating PDO connection to "mysql:host=mysql;port=3306;dbname=testdb;charset=utf8".

tests/functional/error-invalid-input-mapping/source/data/config.json

@@ -0,0 +1,62 @@
+{
+  "parameters": {
+    "writer_class": "Common",
+    "data_dir": "/data",
+    "db": {
+      "driver": "mysql",
+      "host": "%env(string:COMMON_DB_HOST)%",
+      "port": "%env(string:COMMON_DB_PORT)%",
+      "database": "%env(string:COMMON_DB_DATABASE)%",
+      "user": "%env(string:COMMON_DB_USER)%",
+      "#password": "%env(string:COMMON_DB_PASSWORD)%"
+    },
+    "tableId": "invalidtable",
+    "dbName": "simple",
+    "incremental": true,
+    "primaryKey": [
+      "id"
+    ],
+    "items": [
+      {
+        "name": "id",
+        "dbName": "id",
+        "type": "int",
+        "size": null,
+        "nullable": null,
+        "default": null
+      },
+      {
+        "name": "name",
+        "dbName": "name",
+        "type": "VARCHAR",
+        "size": 255,
+        "nullable": null,
+        "default": null
+      },
+      {
+        "name": "glasses",
+        "dbName": "glasses",
+        "type": "VARCHAR",
+        "size": 20,
+        "nullable": null,
+        "default": null
+      }
+    ]
+  },
+  "storage": {
+    "input": {
+      "tables": [
+        {
+          "source": "simple",
+          "destination": "simple.csv",
+          "columns": [
+            "id",
+            "name",
+            "glasses",
+            "age"
+          ]
+        }
+      ]
+    }
+  }
+}

tests/functional/error-invalid-input-mapping/source/data/in/tables/simple.csv

@@ -0,0 +1,10 @@
+"id","name","glasses"
+"0","Miro cillik","yes"
+"1","Ondrej Hlavacek","no"
+"2","Martin Halamicek","yes"
+"3","Tomas Kacur","yes"
+"4","Erik Zigo","no"
+"5","Marc Raiser","sometimes"
+"6","Petr Simecek","yes"
+"7","Jakub Matejka","yes"
+"8","Ondrej Popelka","no"

tests/functional/error-invalid-input-mapping/source/data/in/tables/simple.csv.manifest

@@ -0,0 +1,7 @@
+{
+    "columns": [
+        "id",
+        "name",
+        "glasses"
+    ]
+}

tests/functional/error-ssh-bad-host/expected-code

@@ -0,0 +1 @@
+1

tests/functional/error-ssh-bad-host/expected-stderr

@@ -0,0 +1 @@
+Unable to create ssh tunnel. Output:  ErrorOutput: ssh: Could not resolve hostname unknownhost: Name or service not known Retries count: 5

tests/functional/error-ssh-bad-host/expected-stdout

@@ -0,0 +1,5 @@
+Creating SSH tunnel to 'unknownHost' on local port '33006'
+Unable to create ssh tunnel. Output:  ErrorOutput: ssh: Could not resolve hostname unknownhost: Name or service not known . Retrying... [1x]
+Unable to create ssh tunnel. Output:  ErrorOutput: ssh: Could not resolve hostname unknownhost: Name or service not known . Retrying... [2x]
+Unable to create ssh tunnel. Output:  ErrorOutput: ssh: Could not resolve hostname unknownhost: Name or service not known . Retrying... [3x]
+Unable to create ssh tunnel. Output:  ErrorOutput: ssh: Could not resolve hostname unknownhost: Name or service not known . Retrying... [4x]

tests/functional/error-ssh-bad-host/setUp.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+use Keboola\DbWriter\TestsFunctional\DatadirTest;
+
+return function (DatadirTest $test): void {
+    putenv('SSH_PUBLIC_KEY=' . file_get_contents('/root/.ssh/id_rsa.pub'));
+    putenv('SSH_PRIVATE_KEY=' . file_get_contents('/root/.ssh/id_rsa'));
+};