From 5c6899b33bdb41cee9bcb997093e26349c581fa2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 6 Aug 2025 00:39:49 +1200 Subject: [PATCH 1/9] Add CSV export dest --- src/Migration/Destinations/Appwrite.php | 1 - src/Migration/Destinations/CSV.php | 343 ++++++++++++++++++++++++ 2 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 src/Migration/Destinations/CSV.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 2520acb6..e0b866f3 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -12,7 +12,6 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; -use Override; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php new file mode 100644 index 00000000..09a46a2a --- /dev/null +++ b/src/Migration/Destinations/CSV.php @@ -0,0 +1,343 @@ +deviceForMigrations = $deviceForExports; + $this->resourceId = $resourceId; + $this->local = new Local('/' . $resourceId . '.csv'); + $this->local->setTransferChunkSize(Transfer::STORAGE_MAX_CHUNK_SIZE); + $this->createDirectory($this->local->getRoot()); + + foreach ($allowedAttributes as $attribute) { + $this->allowedAttributes[$attribute] = true; + } + } + + public static function getName(): string + { + return 'CSV'; + } + + public static function getSupportedResources(): array + { + return [ + Resource::TYPE_DOCUMENT, + ]; + } + + public function report(array $resources = []): array + { + return []; + } + + public function shutdown(): void + { + if (!\file_exists($this->local->getRoot())) { + throw new \Exception('Nothing to upload'); + } + + $filename = $this->resourceId . '.csv'; + + try { + $destination = $this->deviceForMigrations->getRoot() . '/' . $filename; + $result = $this->local->transfer($filename, $destination, $this->deviceForMigrations); + + if ($result === false) { + throw new \Exception('Error uploading to ' . $destination); + } + + if (!$this->deviceForMigrations->exists($destination)) { + throw new \Exception('File not found on destination: ' . $destination); + } + } finally { + if (!$this->local->deletePath('') || \file_exists($this->local->getRoot())) { + Console::error('Error deleting: ' . $this->local->getRoot()); + throw new \Exception('Error deleting: ' . $this->local->getRoot()); + } + } + } + + /** + * @param array $resources + * @throws \JsonException + * @throws \Exception + */ + protected function import(array $resources, callable $callback): void + { + $handles = []; // file path => file handle + $buffers = []; // file path => ['lines' => array, 'size' => int] + $bufferBytes = 1024 * 1024; // 1MB + $csvHeaders = []; // file path => bool (to track if headers written) + $csvDataCache = []; // Cache for CSV data to avoid repeated processing + + $flushBuffer = function (string $file) use (&$handles, &$buffers) { + if (empty($buffers[$file]['lines'])) { + return; + } + + try { + if (!isset($handles[$file])) { + $handles[$file] = \fopen($file, 'a'); + if ($handles[$file] === false) { + throw new \Exception("Failed to open file for writing: $file"); + } + } + + $content = \implode('', $buffers[$file]['lines']); + if (\fwrite($handles[$file], $content) === false) { + throw new \Exception("Failed to write to file: $file"); + } + + $buffers[$file] = [ + 'lines' => [], + 'size' => 0 + ]; + } catch (\Exception $e) { + // Close handle on error + if (isset($handles[$file])) { + \fclose($handles[$file]); + unset($handles[$file]); + } + throw $e; + } + }; + + try { + foreach ($resources as $resource) { + $log = $this->local->getRoot() . '/' . $resource->getGroup() . '-' . $resource->getName() . '.csv'; + + if (!isset($buffers[$log])) { + $buffers[$log] = ['lines' => [], 'size' => 0]; + } + + // Write headers if this is the first record for this file + if (!isset($csvHeaders[$log])) { + $csvData = $this->resourceToCSVData($resource); + $csvDataCache[$resource->getId()] = $csvData; + + $headerLine = $this->toCSV(array_keys($csvData)); + $buffers[$log]['lines'][] = $headerLine; + $buffers[$log]['size'] += strlen($headerLine); + $csvHeaders[$log] = true; + } else { + // Use cached data if available, otherwise process + if (!isset($csvDataCache[$resource->getId()])) { + $csvData = $this->resourceToCSVData($resource); + $csvDataCache[$resource->getId()] = $csvData; + } else { + $csvData = $csvDataCache[$resource->getId()]; + } + } + + $dataLine = $this->toCSV(array_values($csvData)); + $buffers[$log]['lines'][] = $dataLine; + $buffers[$log]['size'] += strlen($dataLine); + + if ($buffers[$log]['size'] >= $bufferBytes) { + $flushBuffer($log); + } + + $resource->setStatus(Resource::STATUS_SUCCESS); + if (isset($this->cache)) { + $this->cache->update($resource); + } + } + + // Flush remaining buffers + foreach ($buffers as $file => $bufferData) { + if (!empty($bufferData['lines'])) { + $flushBuffer($file); + } + } + + } finally { + // Ensure all handles are closed + foreach ($handles as $handle) { + if (is_resource($handle)) { + \fclose($handle); + } + } + } + + $callback($resources); + } + + /** + * Helper to ensure a directory exists. + */ + protected function createDirectory(string $path): void + { + if (!\file_exists($path)) { + if (!\mkdir($path, 0755, true)) { + throw new \Exception('Error creating directory: ' . $path); + } + } + } + + /** + * Convert a resource to CSV-compatible data + */ + protected function resourceToCSVData(Document $resource): array + { + $data = [ + '$id' => $resource->getId(), + '$permissions' => $resource->getPermissions(), + ...\array_filter($resource->getData(), function ($key) { + return isset($this->allowedAttributes[$key]); + }, ARRAY_FILTER_USE_KEY) + ]; + + $results = []; + + foreach ($data as $key => $value) { + $results[$key] = $this->convertValueToCSV($value); + } + + return $results; + } + + /** + * Convert a single value to CSV-compatible format + */ + protected function convertValueToCSV(mixed $value): string + { + if (\is_array($value)) { + return $this->convertMapToCSV($value); + } + if (\is_object($value)) { + return $this->convertObjectToCSV($value); + } + return $this->escape($value); + } + + /** + * Convert array to CSV format + */ + protected function convertMapToCSV(array $value): string + { + if (empty($value)) { + return '""'; + } + if (isset($value['$id'])) { + return $this->escape($value['$id']); + } + if (!\array_is_list($value)) { + return $this->escape(\json_encode($value)); + } + return $this->convertListToCSV($value); + } + + /** + * Convert indexed array to CSV format + */ + protected function convertListToCSV(array $value): string + { + $count = \count($value); + if ($count === 0) { + return '""'; + } + + $processed = []; + for ($i = 0; $i < $count; $i++) { + if (\is_array($value[$i]) && isset($value['$id'])) { + $processed[] = $value['$id']; + continue; + } + $processed[] = $this->escape($value[$i]); + } + + return '"' . implode(',', $processed) . '"'; + } + + /** + * Convert object to CSV format + */ + protected function convertObjectToCSV($value): string + { + if ($value instanceof Document) { + return $this->escape($value->getId()); + } + return $this->escape(\json_encode($value)); + } + + /** + * Convert array to CSV line with proper escaping + */ + protected function toCSV(array $array): string + { + if (empty($array)) { + return "\n"; + } + + $line = $this->escape($array[0]); + $count = \count($array); + + for ($i = 1; $i < $count; $i++) { + $line .= ',' . $this->escape($array[$i]); + } + + return $line . "\n"; + } + + /** + * Safely escape a value for CSV (backslash-escape style) + * - null/empty -> empty string + * - bool -> true/false + * - numeric -> raw + * - strings with special chars -> quoted with backslash escapes + */ + protected function escape($value): string + { + if (\is_null($value) || $value === '') { + return ''; + } + if (\is_bool($value)) { + return $value ? 'true' : 'false'; + } + if (\is_numeric($value)) { + return (string)$value; + } + + $stringValue = (string)$value; + + // Escape backslashes first, then quotes + $escaped = \str_replace(['\\', '"'], ['\\\\', '\\"'], $stringValue); + + // Needs quoting if it contains commas, line breaks, quotes, or backslashes + if (\strpbrk($stringValue, ",\n\r\"\\") !== false) { + return '"' . $escaped . '"'; + } + + return $escaped; + } +} From 8435f1db0db4854ca27cb4c9cf275b905fcb3b41 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 00:56:18 +1200 Subject: [PATCH 2/9] Fix export --- src/Migration/Destinations/CSV.php | 280 +++++++---------- src/Migration/Resources/Database/Row.php | 7 +- tests/Migration/Unit/General/CSVTest.php | 372 +++++++++++++++++++++++ 3 files changed, 490 insertions(+), 169 deletions(-) diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index 09a46a2a..250231b5 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -8,7 +8,7 @@ use Utopia\Database\Exception\Structure; use Utopia\Migration\Destination; use Utopia\Migration\Resource; -use Utopia\Migration\Resources\Database\Document; +use Utopia\Migration\Resources\Database\Row; use Utopia\Migration\Transfer; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -34,7 +34,7 @@ public function __construct( ) { $this->deviceForMigrations = $deviceForExports; $this->resourceId = $resourceId; - $this->local = new Local('/' . $resourceId . '.csv'); + $this->local = new Local(\sys_get_temp_dir() . '/csv_export_' . uniqid()); $this->local->setTransferChunkSize(Transfer::STORAGE_MAX_CHUNK_SIZE); $this->createDirectory($this->local->getRoot()); @@ -51,7 +51,7 @@ public static function getName(): string public static function getSupportedResources(): array { return [ - Resource::TYPE_DOCUMENT, + Resource::TYPE_ROW, ]; } @@ -60,73 +60,44 @@ public function report(array $resources = []): array return []; } - public function shutdown(): void - { - if (!\file_exists($this->local->getRoot())) { - throw new \Exception('Nothing to upload'); - } - - $filename = $this->resourceId . '.csv'; - - try { - $destination = $this->deviceForMigrations->getRoot() . '/' . $filename; - $result = $this->local->transfer($filename, $destination, $this->deviceForMigrations); - - if ($result === false) { - throw new \Exception('Error uploading to ' . $destination); - } - - if (!$this->deviceForMigrations->exists($destination)) { - throw new \Exception('File not found on destination: ' . $destination); - } - } finally { - if (!$this->local->deletePath('') || \file_exists($this->local->getRoot())) { - Console::error('Error deleting: ' . $this->local->getRoot()); - throw new \Exception('Error deleting: ' . $this->local->getRoot()); - } - } - } - /** - * @param array $resources + * @param array $resources * @throws \JsonException * @throws \Exception */ protected function import(array $resources, callable $callback): void { - $handles = []; // file path => file handle - $buffers = []; // file path => ['lines' => array, 'size' => int] + $handle = null; // file handle + $buffer = ['lines' => [], 'size' => 0]; // Buffer for batching writes $bufferBytes = 1024 * 1024; // 1MB - $csvHeaders = []; // file path => bool (to track if headers written) - $csvDataCache = []; // Cache for CSV data to avoid repeated processing + $log = $this->local->getRoot() . '/' . $this->resourceId . '.csv'; - $flushBuffer = function (string $file) use (&$handles, &$buffers) { - if (empty($buffers[$file]['lines'])) { + $flushBuffer = function () use ($log, &$handle, &$buffer) { + if (empty($buffer['lines'])) { return; } - try { - if (!isset($handles[$file])) { - $handles[$file] = \fopen($file, 'a'); - if ($handles[$file] === false) { - throw new \Exception("Failed to open file for writing: $file"); + if (!isset($handle)) { + $handle = \fopen($log, 'a'); + if ($handle === false) { + throw new \Exception("Failed to open file for writing: $log"); } } - $content = \implode('', $buffers[$file]['lines']); - if (\fwrite($handles[$file], $content) === false) { - throw new \Exception("Failed to write to file: $file"); + $content = \implode('', $buffer['lines']); + if (\fwrite($handle, $content) === false) { + throw new \Exception("Failed to write to file: $log"); } - $buffers[$file] = [ + $buffer = [ 'lines' => [], 'size' => 0 ]; } catch (\Exception $e) { // Close handle on error - if (isset($handles[$file])) { - \fclose($handles[$file]); - unset($handles[$file]); + if (isset($handle)) { + \fclose($handle); + unset($handle); } throw $e; } @@ -134,37 +105,22 @@ protected function import(array $resources, callable $callback): void try { foreach ($resources as $resource) { - $log = $this->local->getRoot() . '/' . $resource->getGroup() . '-' . $resource->getName() . '.csv'; - - if (!isset($buffers[$log])) { - $buffers[$log] = ['lines' => [], 'size' => 0]; - } - - // Write headers if this is the first record for this file - if (!isset($csvHeaders[$log])) { - $csvData = $this->resourceToCSVData($resource); - $csvDataCache[$resource->getId()] = $csvData; - - $headerLine = $this->toCSV(array_keys($csvData)); - $buffers[$log]['lines'][] = $headerLine; - $buffers[$log]['size'] += strlen($headerLine); - $csvHeaders[$log] = true; - } else { - // Use cached data if available, otherwise process - if (!isset($csvDataCache[$resource->getId()])) { - $csvData = $this->resourceToCSVData($resource); - $csvDataCache[$resource->getId()] = $csvData; - } else { - $csvData = $csvDataCache[$resource->getId()]; - } + $csvData = $this->resourceToCSVData($resource); + + // Write headers if this is the first row of the file + if (!isset($csvHeader)) { + $headers = $this->toCSV(\array_keys($csvData)); + $buffer['lines'][] = $headers; + $buffer['size'] += \strlen($headers); + $csvHeader = true; } - $dataLine = $this->toCSV(array_values($csvData)); - $buffers[$log]['lines'][] = $dataLine; - $buffers[$log]['size'] += strlen($dataLine); + $dataLine = $this->toCSV(\array_values($csvData)); + $buffer['lines'][] = $dataLine; + $buffer['size'] += \strlen($dataLine); - if ($buffers[$log]['size'] >= $bufferBytes) { - $flushBuffer($log); + if ($buffer['size'] >= $bufferBytes) { + $flushBuffer(); } $resource->setStatus(Resource::STATUS_SUCCESS); @@ -173,27 +129,57 @@ protected function import(array $resources, callable $callback): void } } - // Flush remaining buffers - foreach ($buffers as $file => $bufferData) { - if (!empty($bufferData['lines'])) { - $flushBuffer($file); - } + // Flush any remaining buffered lines + if (!empty($buffer['lines'])) { + $flushBuffer(); } - } finally { - // Ensure all handles are closed - foreach ($handles as $handle) { - if (is_resource($handle)) { - \fclose($handle); - } + if (\is_resource($handle)) { + \fclose($handle); } } $callback($resources); } + /** + * @throws \Exception + */ + public function shutdown(): void + { + if (!\file_exists($this->local->getRoot())) { + throw new \Exception('Nothing to upload'); + } + + $filename = $this->resourceId . '.csv'; + + try { + // Transfer expects relative paths within each device + $result = $this->local->transfer( + $filename, + $filename, + $this->deviceForMigrations + ); + + if ($result === false) { + throw new \Exception('Error uploading to ' . $this->deviceForMigrations->getRoot() . '/' . $filename); + } + + if (!$this->deviceForMigrations->exists($filename)) { + throw new \Exception('File not found on destination: ' . $filename); + } + } finally { + // Clean up the temporary directory + if (!$this->local->deletePath('') || \file_exists($this->local->getRoot())) { + Console::error('Error deleting: ' . $this->local->getRoot()); + throw new \Exception('Error deleting: ' . $this->local->getRoot()); + } + } + } + /** * Helper to ensure a directory exists. + * @throws \Exception */ protected function createDirectory(string $path): void { @@ -207,23 +193,29 @@ protected function createDirectory(string $path): void /** * Convert a resource to CSV-compatible data */ - protected function resourceToCSVData(Document $resource): array + protected function resourceToCSVData(Row $resource): array { $data = [ '$id' => $resource->getId(), '$permissions' => $resource->getPermissions(), - ...\array_filter($resource->getData(), function ($key) { - return isset($this->allowedAttributes[$key]); - }, ARRAY_FILTER_USE_KEY) ]; - - $results = []; + + // Add all attributes if no filter specified, otherwise only allowed ones + if (empty($this->allowedAttributes)) { + $data = \array_merge($data, $resource->getData()); + } else { + foreach ($resource->getData() as $key => $value) { + if (isset($this->allowedAttributes[$key])) { + $data[$key] = $value; + } + } + } foreach ($data as $key => $value) { - $results[$key] = $this->convertValueToCSV($value); + $data[$key] = $this->convertValueToCSV($value); } - return $results; + return $data; } /** @@ -231,52 +223,33 @@ protected function resourceToCSVData(Document $resource): array */ protected function convertValueToCSV(mixed $value): string { + if (\is_null($value)) { + return 'null'; + } + if (\is_bool($value)) { + return $value ? 'true' : 'false'; + } if (\is_array($value)) { - return $this->convertMapToCSV($value); + return $this->convertArrayToCSV($value); } if (\is_object($value)) { return $this->convertObjectToCSV($value); } - return $this->escape($value); + return (string)$value; } /** * Convert array to CSV format */ - protected function convertMapToCSV(array $value): string + protected function convertArrayToCSV(array $value): string { if (empty($value)) { - return '""'; + return ''; } if (isset($value['$id'])) { - return $this->escape($value['$id']); - } - if (!\array_is_list($value)) { - return $this->escape(\json_encode($value)); - } - return $this->convertListToCSV($value); - } - - /** - * Convert indexed array to CSV format - */ - protected function convertListToCSV(array $value): string - { - $count = \count($value); - if ($count === 0) { - return '""'; - } - - $processed = []; - for ($i = 0; $i < $count; $i++) { - if (\is_array($value[$i]) && isset($value['$id'])) { - $processed[] = $value['$id']; - continue; - } - $processed[] = $this->escape($value[$i]); + return $value['$id']; } - - return '"' . implode(',', $processed) . '"'; + return \json_encode($value); } /** @@ -284,60 +257,35 @@ protected function convertListToCSV(array $value): string */ protected function convertObjectToCSV($value): string { - if ($value instanceof Document) { - return $this->escape($value->getId()); + if ($value instanceof Row) { + return $value->getId(); } - return $this->escape(\json_encode($value)); + return \json_encode($value); } /** * Convert array to CSV line with proper escaping + * Uses standard CSV format with double-quote escaping */ protected function toCSV(array $array): string { - if (empty($array)) { - return "\n"; - } - - $line = $this->escape($array[0]); - $count = \count($array); - - for ($i = 1; $i < $count; $i++) { - $line .= ',' . $this->escape($array[$i]); + $output = []; + foreach ($array as $value) { + $output[] = $this->escapeForCSV($value); } - - return $line . "\n"; + return \implode(',', $output) . "\n"; } /** - * Safely escape a value for CSV (backslash-escape style) - * - null/empty -> empty string - * - bool -> true/false - * - numeric -> raw - * - strings with special chars -> quoted with backslash escapes + * Escape a single value for CSV format */ - protected function escape($value): string + protected function escapeForCSV(string $value): string { - if (\is_null($value) || $value === '') { - return ''; - } - if (\is_bool($value)) { - return $value ? 'true' : 'false'; - } - if (\is_numeric($value)) { - return (string)$value; - } - - $stringValue = (string)$value; - - // Escape backslashes first, then quotes - $escaped = \str_replace(['\\', '"'], ['\\\\', '\\"'], $stringValue); - - // Needs quoting if it contains commas, line breaks, quotes, or backslashes - if (\strpbrk($stringValue, ",\n\r\"\\") !== false) { + if (\strpbrk($value, ",\n\r\"") !== false) { + // Escape quotes by doubling them (CSV standard) + $escaped = \str_replace('"', '""', $value); return '"' . $escaped . '"'; } - - return $escaped; + return $value; } } diff --git a/src/Migration/Resources/Database/Row.php b/src/Migration/Resources/Database/Row.php index cebd88e7..6b7e9ada 100644 --- a/src/Migration/Resources/Database/Row.php +++ b/src/Migration/Resources/Database/Row.php @@ -14,11 +14,12 @@ class Row extends Resource * @param array $permissions */ public function __construct( - string $id, + string $id, private readonly Table $table, private readonly array $data = [], - array $permissions = [] - ) { + array $permissions = [] + ) + { $this->id = $id; $this->permissions = $permissions; } diff --git a/tests/Migration/Unit/General/CSVTest.php b/tests/Migration/Unit/General/CSVTest.php index 7cae1c7b..3414e5fb 100644 --- a/tests/Migration/Unit/General/CSVTest.php +++ b/tests/Migration/Unit/General/CSVTest.php @@ -4,6 +4,33 @@ use PHPUnit\Framework\TestCase; use Utopia\Migration\Sources\CSV; +use Utopia\Migration\Destinations\CSV as DestinationCSV; +use Utopia\Migration\Resources\Database\Row; +use Utopia\Migration\Resources\Database\Table; +use Utopia\Migration\Resources\Database\Database; +use Utopia\Storage\Device\Local; + +/** + * Test-friendly CSV destination + */ +class TestCSV extends DestinationCSV +{ + public function testableImport(array $resources, callable $callback): void + { + $this->import($resources, $callback); + } + + public function getLocalRoot(): string + { + return $this->local->getRoot(); + } + + // Override shutdown to avoid transfer for testing + public function shutdown(): void + { + // Do nothing for testing - don't transfer files + } +} class CSVTest extends TestCase { @@ -46,4 +73,349 @@ public function testDetectDelimiter() $this->assertEquals($case['expected'], $delimiter, "Failed for {$case['file']}"); } } + + public function testCSVExportBasic() + { + $tempDir = sys_get_temp_dir() . '/csv_test_' . uniqid(); + mkdir($tempDir, 0755, true); + $exportDevice = new Local($tempDir); + + // Create CSV destination + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + + // Create test data + $database = new Database('test_db'); + $table = new Table($database, 'test_table', 'test_table_id'); + + $row1 = new Row('row1', $table, [ + 'name' => 'John Doe', + 'age' => 30, + 'email' => 'john@example.com' + ]); + $row1->setPermissions(['read' => ['user:123']]); + + $row2 = new Row('row2', $table, [ + 'name' => 'Jane Smith', + 'age' => 25, + 'email' => 'jane@example.com' + ]); + $row2->setPermissions(['read' => ['user:456']]); + + // Export the data + $csvDestination->testableImport([$row1, $row2], function($resources) { + // Callback - verify resources are marked as successful + foreach ($resources as $resource) { + $this->assertEquals('success', $resource->getStatus()); + } + }); + + $csvDestination->shutdown(); + + // Verify CSV file was created in local temp directory + $expectedFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + $this->assertFileExists($expectedFile, 'CSV file should exist'); + + // Use proper CSV parsing + $handle = fopen($expectedFile, 'r'); + $this->assertNotFalse($handle); + + $header = fgetcsv($handle, 0, ',', '"', '"'); + $row1Data = fgetcsv($handle, 0, ',', '"', '"'); + $row2Data = fgetcsv($handle, 0, ',', '"', '"'); + fclose($handle); + + $this->assertNotFalse($header); + $this->assertNotFalse($row1Data); + $this->assertNotFalse($row2Data); + + // Check header + $this->assertContains('$id', $header); + $this->assertContains('$permissions', $header); + $this->assertContains('name', $header); + $this->assertContains('age', $header); + $this->assertContains('email', $header); + + // Check first row data + $this->assertEquals('row1', $row1Data[0]); // $id + $this->assertStringContainsString('user:123', $row1Data[1]); // $permissions + $this->assertEquals('John Doe', $row1Data[2]); // name + $this->assertEquals('30', $row1Data[3]); // age + $this->assertEquals('john@example.com', $row1Data[4]); // email + + // Cleanup + if (is_dir($tempDir)) { + $this->recursiveDelete($tempDir); + } + } + + public function testCSVExportWithSpecialCharacters() + { + $tempDir = sys_get_temp_dir() . '/csv_test_special_' . uniqid(); + $exportDevice = new Local($tempDir); + + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + + $database = new Database('test_db'); + $table = new Table($database, 'test_table', 'test_table_id'); + + // Test data with special characters that need escaping + $row = new Row('special_row', $table, [ + 'quote_field' => 'Text with "quotes"', + 'comma_field' => 'Text, with, commas', + 'newline_field' => "Text with\nnewlines", + 'mixed_field' => 'Text with "quotes", commas, and\nnewlines' + ]); + + $csvDestination->testableImport([$row], function($resources) {}); + $csvDestination->shutdown(); + + $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + + // Use proper CSV parsing + $handle = fopen($csvFile, 'r'); + $this->assertNotFalse($handle); + + $header = fgetcsv($handle, 0, ',', '"', '"'); + $rowData = fgetcsv($handle, 0, ',', '"', '"'); + fclose($handle); + + $this->assertNotFalse($header); + $this->assertNotFalse($rowData); + + // Verify special characters are properly handled (fgetcsv($handle, 0, ',', '"', '"'); + $this->assertEquals('Text with "quotes"', $rowData[2]); // quote_field + $this->assertEquals('Text, with, commas', $rowData[3]); // comma_field + $this->assertEquals("Text with\nnewlines", $rowData[4]); // newline_field + $this->assertEquals('Text with "quotes", commas, and\nnewlines', $rowData[5]); // mixed_field + + // Cleanup + if (is_dir($tempDir)) { + $this->recursiveDelete($tempDir); + } + } + + public function testCSVExportWithArrays() + { + $tempDir = sys_get_temp_dir() . '/csv_test_arrays_' . uniqid(); + $exportDevice = new Local($tempDir); + + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + + $database = new Database('test_db'); + $table = new Table($database, 'test_table', 'test_table_id'); + + $row = new Row('array_row', $table, [ + 'tags' => ['php', 'csv', 'export'], + 'metadata' => ['key1' => 'value1', 'key2' => 'value2'], + 'empty_array' => [], + 'nested' => [['id' => 1], ['id' => 2]] + ]); + + $csvDestination->testableImport([$row], function($resources) {}); + $csvDestination->shutdown(); + + $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + + // Use proper CSV parsing + $handle = fopen($csvFile, 'r'); + $this->assertNotFalse($handle); + + $header = fgetcsv($handle, 0, ',', '"', '"'); + $rowData = fgetcsv($handle, 0, ',', '"', '"'); + fclose($handle); + + $this->assertNotFalse($header); + $this->assertNotFalse($rowData); + + // Arrays should be JSON encoded + $this->assertEquals('["php","csv","export"]', $rowData[2]); // tags + $this->assertJson($rowData[3]); // metadata should be valid JSON + $this->assertEquals('', $rowData[4]); // empty_array + $this->assertJson($rowData[5]); // nested should be valid JSON + + // Cleanup + if (is_dir($tempDir)) { + $this->recursiveDelete($tempDir); + } + } + + public function testCSVExportWithNullValues() + { + $tempDir = sys_get_temp_dir() . '/csv_test_nulls_' . uniqid(); + $exportDevice = new Local($tempDir); + + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + + $database = new Database('test_db'); + $table = new Table($database, 'test_table', 'test_table_id'); + + $row = new Row('null_row', $table, [ + 'name' => 'Test', + 'null_field' => null, + 'empty_string' => '', + 'zero' => 0, + 'false_bool' => false + ]); + + $csvDestination->testableImport([$row], function($resources) {}); + $csvDestination->shutdown(); + + $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + + // Use proper CSV parsing + $handle = fopen($csvFile, 'r'); + $this->assertNotFalse($handle); + + $header = fgetcsv($handle, 0, ',', '"', '"'); + $rowData = fgetcsv($handle, 0, ',', '"', '"'); + fclose($handle); + + $this->assertNotFalse($header); + $this->assertNotFalse($rowData); + + $this->assertEquals('Test', $rowData[2]); // name + $this->assertEquals('null', $rowData[3]); // null_field -> "null" string + $this->assertEquals('', $rowData[4]); // empty_string + $this->assertEquals('0', $rowData[5]); // zero + $this->assertEquals('false', $rowData[6]); // false_bool + + // Cleanup + if (is_dir($tempDir)) { + $this->recursiveDelete($tempDir); + } + } + + public function testCSVExportWithAllowedAttributes() + { + $tempDir = sys_get_temp_dir() . '/csv_test_filtered_' . uniqid(); + $exportDevice = new Local($tempDir); + + // Only allow specific attributes + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', ['name', 'email']); + + $database = new Database('test_db'); + $table = new Table($database, 'test_table', 'test_table_id'); + + $row = new Row('filtered_row', $table, [ + 'name' => 'John Doe', + 'age' => 30, + 'email' => 'john@example.com', + 'secret' => 'should_not_appear' + ]); + + $csvDestination->testableImport([$row], function($resources) {}); + $csvDestination->shutdown(); + + $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + + // Use proper CSV parsing + $handle = fopen($csvFile, 'r'); + $this->assertNotFalse($handle); + + $header = fgetcsv($handle, 0, ',', '"', '"'); + $rowData = fgetcsv($handle, 0, ',', '"', '"'); + fclose($handle); + + $this->assertNotFalse($header); + $this->assertNotFalse($rowData); + + // Should have $id, $permissions, and only allowed attributes + $this->assertContains('$id', $header); + $this->assertContains('$permissions', $header); + $this->assertContains('name', $header); + $this->assertContains('email', $header); + $this->assertNotContains('age', $header); + $this->assertNotContains('secret', $header); + + // Cleanup + if (is_dir($tempDir)) { + $this->recursiveDelete($tempDir); + } + } + + public function testCSVExportImportCompatibility() + { + $tempDir = sys_get_temp_dir() . '/csv_test_compat_' . uniqid(); + $exportDevice = new Local($tempDir); + + // Export data + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + + $database = new Database('test_db'); + $table = new Table($database, 'test_table', 'test_table_id'); + + $originalData = [ + 'name' => 'John Doe', + 'age' => 30, + 'tags' => ['php', 'csv'], + 'metadata' => ['key' => 'value'], + 'null_field' => null, + 'empty_field' => '', + 'bool_field' => true + ]; + + $row = new Row('compat_row', $table, $originalData); + $row->setPermissions(['read' => ['user:123']]); + + $csvDestination->testableImport([$row], function($resources) {}); + $csvDestination->shutdown(); + + // Verify the exported CSV can be parsed by PHP's built-in CSV functions + $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + $this->assertFileExists($csvFile); + + $handle = fopen($csvFile, 'r'); + $this->assertNotFalse($handle); + + $header = fgetcsv($handle, 0, ',', '"', '"'); + $data = fgetcsv($handle, 0, ',', '"', '"'); + fclose($handle); + + $this->assertNotFalse($header); + $this->assertNotFalse($data); + + // Verify we can reconstruct the data + $reconstructed = \array_combine($header, $data); + + $this->assertEquals('compat_row', $reconstructed['$id']); + $this->assertEquals('John Doe', $reconstructed['name']); + $this->assertEquals('30', $reconstructed['age']); + $this->assertEquals('null', $reconstructed['null_field']); // null becomes "null" string + $this->assertEquals('', $reconstructed['empty_field']); + $this->assertEquals('true', $reconstructed['bool_field']); // bool becomes string + + // Arrays should be valid JSON that can be decoded + $this->assertJson($reconstructed['tags']); + $this->assertJson($reconstructed['metadata']); + + $tagsArray = json_decode($reconstructed['tags'], true); + $metadataArray = json_decode($reconstructed['metadata'], true); + + $this->assertEquals(['php', 'csv'], $tagsArray); + $this->assertEquals(['key' => 'value'], $metadataArray); + + // Cleanup + if (is_dir($tempDir)) { + $this->recursiveDelete($tempDir); + } + } + + private function recursiveDelete(string $dir): void + { + if (is_dir($dir)) { + $objects = scandir($dir); + if ($objects !== false) { + foreach ($objects as $object) { + if ($object != "." && $object != "..") { + if (is_dir($dir."/".$object)) { + $this->recursiveDelete($dir."/".$object); + } else { + unlink($dir."/".$object); + } + } + } + } + rmdir($dir); + } + } } From fd365e147bface646ac552c1ffc062874deb51df Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 19:55:46 +1200 Subject: [PATCH 3/9] Fix file paths --- src/Migration/Destinations/CSV.php | 61 +++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index 250231b5..8042055a 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -17,9 +17,10 @@ class CSV extends Destination { protected Device $deviceForMigrations; protected string $resourceId; + protected string $outputFile; protected Local $local; - protected array $allowedAttributes = []; + protected array $allowedColumns = []; /** * @throws Authorization @@ -30,16 +31,17 @@ class CSV extends Destination public function __construct( Device $deviceForExports, string $resourceId, - array $allowedAttributes = [] + array $allowedColumns = [] ) { $this->deviceForMigrations = $deviceForExports; $this->resourceId = $resourceId; + $this->outputFile = $this->sanitizeFilename($resourceId); $this->local = new Local(\sys_get_temp_dir() . '/csv_export_' . uniqid()); $this->local->setTransferChunkSize(Transfer::STORAGE_MAX_CHUNK_SIZE); $this->createDirectory($this->local->getRoot()); - foreach ($allowedAttributes as $attribute) { - $this->allowedAttributes[$attribute] = true; + foreach ($allowedColumns as $attribute) { + $this->allowedColumns[$attribute] = true; } } @@ -70,7 +72,7 @@ protected function import(array $resources, callable $callback): void $handle = null; // file handle $buffer = ['lines' => [], 'size' => 0]; // Buffer for batching writes $bufferBytes = 1024 * 1024; // 1MB - $log = $this->local->getRoot() . '/' . $this->resourceId . '.csv'; + $log = $this->local->getRoot() . '/' . $this->outputFile . '.csv'; $flushBuffer = function () use ($log, &$handle, &$buffer) { if (empty($buffer['lines'])) { @@ -105,6 +107,10 @@ protected function import(array $resources, callable $callback): void try { foreach ($resources as $resource) { + if (!($resource instanceof Row)) { + continue; + } + $csvData = $this->resourceToCSVData($resource); // Write headers if this is the first row of the file @@ -147,13 +153,20 @@ protected function import(array $resources, callable $callback): void */ public function shutdown(): void { - if (!\file_exists($this->local->getRoot())) { - throw new \Exception('Nothing to upload'); - } + $filename = $this->outputFile . '.csv'; + $localFilePath = $this->local->getRoot() . '/' . $filename; - $filename = $this->resourceId . '.csv'; + // Check if the CSV file was actually created + if (!$this->local->exists($localFilePath)) { + throw new \Exception("No data to export for resource: $this->resourceId"); + } try { + $destRoot = $this->deviceForMigrations->getRoot(); + if (!$this->deviceForMigrations->exists($destRoot)) { + $this->deviceForMigrations->createDirectory($destRoot); + } + // Transfer expects relative paths within each device $result = $this->local->transfer( $filename, @@ -165,14 +178,16 @@ public function shutdown(): void throw new \Exception('Error uploading to ' . $this->deviceForMigrations->getRoot() . '/' . $filename); } - if (!$this->deviceForMigrations->exists($filename)) { - throw new \Exception('File not found on destination: ' . $filename); + $finalDestPath = $destRoot . '/' . $filename; + if (!$this->deviceForMigrations->exists($finalDestPath)) { + throw new \Exception('File not found on destination: ' . $finalDestPath); + } else { + Console::info("Export successful! File created at: $finalDestPath"); } } finally { // Clean up the temporary directory - if (!$this->local->deletePath('') || \file_exists($this->local->getRoot())) { - Console::error('Error deleting: ' . $this->local->getRoot()); - throw new \Exception('Error deleting: ' . $this->local->getRoot()); + if (!$this->local->deletePath('') || $this->local->exists($this->local->getRoot())) { + Console::error('Error cleaning up: ' . $this->local->getRoot()); } } } @@ -190,6 +205,18 @@ protected function createDirectory(string $path): void } } + /** + * Sanitize a filename to make it filesystem-safe + */ + protected function sanitizeFilename(string $filename): string + { + // Replace problematic characters with underscores + $sanitized = \preg_replace('/[:\\/<>"|*?]/', '_', $filename); + $sanitized = \preg_replace('/[^\x20-\x7E]/', '_', $sanitized); + $sanitized = \trim($sanitized); + return empty($sanitized) ? 'export' : $sanitized; + } + /** * Convert a resource to CSV-compatible data */ @@ -199,13 +226,13 @@ protected function resourceToCSVData(Row $resource): array '$id' => $resource->getId(), '$permissions' => $resource->getPermissions(), ]; - + // Add all attributes if no filter specified, otherwise only allowed ones - if (empty($this->allowedAttributes)) { + if (empty($this->allowedColumns)) { $data = \array_merge($data, $resource->getData()); } else { foreach ($resource->getData() as $key => $value) { - if (isset($this->allowedAttributes[$key])) { + if (isset($this->allowedColumns[$key])) { $data[$key] = $value; } } From ad949420c678328ef612dcaeb25ed66c98213cbd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 19:57:35 +1200 Subject: [PATCH 4/9] Fix processing composite resource IDs --- src/Migration/Sources/Appwrite.php | 45 ++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 69479b16..bd21dcd2 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -707,7 +707,17 @@ private function exportDatabases(int $batchSize): void $queries = [$this->database->queryLimit($batchSize)]; if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { - $queries[] = $this->database->queryEqual('$id', [$this->rootResourceId]); + $targetDatabaseId = $this->rootResourceId; + + // Handle database:collection format - extract database ID + if (\str_contains($this->rootResourceId, ':')) { + $parts = \explode(':', $this->rootResourceId, 2); + if (\count($parts) === 2) { + $targetDatabaseId = $parts[0]; + } + } + + $queries[] = $this->database->queryEqual('$id', [$targetDatabaseId]); $queries[] = $this->database->queryLimit(1); } @@ -735,11 +745,11 @@ private function exportDatabases(int $batchSize): void break; } - $lastDatabase = $databases[count($databases) - 1]; + $lastDatabase = $databases[\count($databases) - 1]; $this->callback($databases); - if (count($databases) < $batchSize) { + if (\count($databases) < $batchSize) { break; } } @@ -754,14 +764,33 @@ private function exportTables(int $batchSize): void $databases = $this->cache->get(Database::getName()); foreach ($databases as $database) { + /** @var Database $database */ $lastTable = null; - /** @var Database $database */ while (true) { $queries = [$this->database->queryLimit($batchSize)]; $tables = []; - if ($lastTable) { + // Filter to specific table if rootResourceType is database with database:collection format + if ( + $this->rootResourceId !== '' && + $this->rootResourceType === Resource::TYPE_DATABASE && + \str_contains($this->rootResourceId, ':') + ) { + $parts = \explode(':', $this->rootResourceId, 2); + if (\count($parts) === 2) { + $targetTableId = $parts[1]; // table ID + $queries[] = $this->database->queryEqual('$id', [$targetTableId]); + $queries[] = $this->database->queryLimit(1); + } + } elseif ( + $this->rootResourceId !== '' && + $this->rootResourceType === Resource::TYPE_TABLE + ) { + $targetTableId = $this->rootResourceId; + $queries[] = $this->database->queryEqual('$id', [$targetTableId]); + $queries[] = $this->database->queryLimit(1); + } elseif ($lastTable) { $queries[] = $this->database->queryCursorAfter($lastTable); } @@ -787,9 +816,9 @@ private function exportTables(int $batchSize): void $this->callback($tables); - $lastTable = $tables[count($tables) - 1]; + $lastTable = $tables[\count($tables) - 1]; - if (count($tables) < $batchSize) { + if (\count($tables) < $batchSize) { break; } } @@ -804,7 +833,7 @@ private function exportColumns(int $batchSize): void { $tables = $this->cache->get(Table::getName()); - /** @var Table[] $tables */ + /** @var array $tables */ foreach ($tables as $table) { $lastColumn = null; From 34edcd472b17f0d98812d429c1ca2eb903d015c1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 22 Aug 2025 20:25:53 +1200 Subject: [PATCH 5/9] Fix paths --- src/Migration/Destinations/CSV.php | 23 +++++++---------------- src/Migration/Sources/CSV.php | 13 +++++++------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index 8042055a..ccbe1ae4 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -154,35 +154,26 @@ protected function import(array $resources, callable $callback): void public function shutdown(): void { $filename = $this->outputFile . '.csv'; - $localFilePath = $this->local->getRoot() . '/' . $filename; + $sourceFilePath = $this->local->getPath($filename); + $destFilePath = $this->deviceForMigrations->getPath($filename); // Check if the CSV file was actually created - if (!$this->local->exists($localFilePath)) { + if (!$this->local->exists($sourceFilePath)) { throw new \Exception("No data to export for resource: $this->resourceId"); } try { - $destRoot = $this->deviceForMigrations->getRoot(); - if (!$this->deviceForMigrations->exists($destRoot)) { - $this->deviceForMigrations->createDirectory($destRoot); - } - - // Transfer expects relative paths within each device + // Transfer expects absolute paths within each device $result = $this->local->transfer( $filename, $filename, $this->deviceForMigrations ); - if ($result === false) { - throw new \Exception('Error uploading to ' . $this->deviceForMigrations->getRoot() . '/' . $filename); + throw new \Exception('Error transferring to ' . $this->deviceForMigrations->getRoot() . '/' . $filename); } - - $finalDestPath = $destRoot . '/' . $filename; - if (!$this->deviceForMigrations->exists($finalDestPath)) { - throw new \Exception('File not found on destination: ' . $finalDestPath); - } else { - Console::info("Export successful! File created at: $finalDestPath"); + if (!$this->deviceForMigrations->exists($destFilePath)) { + throw new \Exception('File not found on destination: ' . $destFilePath); } } finally { // Clean up the temporary directory diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index f43cb046..7ccc37ea 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -131,12 +131,13 @@ private function exportRows(int $batchSize): void $columns = []; $lastColumn = null; - [$databaseId, $tableId] = explode(':', $this->resourceId); + [$databaseId, $tableId] = \explode(':', $this->resourceId); $database = new Database($databaseId, ''); $table = new Table($database, '', $tableId); while (true) { $queries = [$this->database->queryLimit($batchSize)]; + if ($lastColumn) { $queries[] = $this->database->queryCursorAfter($lastColumn); } @@ -146,10 +147,10 @@ private function exportRows(int $batchSize): void break; } - array_push($columns, ...$fetched); - $lastColumn = $fetched[count($fetched) - 1]; + \array_push($columns, ...$fetched); + $lastColumn = $fetched[\count($fetched) - 1]; - if (count($fetched) < $batchSize) { + if (\count($fetched) < $batchSize) { break; } } @@ -206,7 +207,7 @@ private function exportRows(int $batchSize): void $buffer = []; while (($row = \fgetcsv($stream, 0, $delimiter, '"', '"')) !== false) { - if (count($row) !== count($headers)) { + if (\count($row) !== \count($headers)) { throw new \Exception('CSV row does not match the number of header columns.'); } @@ -273,7 +274,7 @@ private function exportRows(int $batchSize): void /** * Parsing logic for best compatibility with spec and 3rd party tools. - * - 'null' unquoted literal string is converted to null. + * - 'null' unquoted literal is converted to null. * - missing strings stay empty strings for best compatibility. * - missing numbers, booleans, and datetime's are converted to null. * - other values are parsed as per their type. From 08c680d5af67deae8274fae3f43a9ffa4669f96b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 15 Sep 2025 16:57:02 +1200 Subject: [PATCH 6/9] Lint --- src/Migration/Destinations/Appwrite.php | 1 + src/Migration/Destinations/CSV.php | 54 +++---- src/Migration/Resources/Database/Row.php | 3 +- tests/Migration/Unit/General/CSVTest.php | 183 ++++++++++++----------- 4 files changed, 112 insertions(+), 129 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 7e52df33..4e38457e 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -12,6 +12,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Override; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index ccbe1ae4..76d560f6 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -31,7 +31,11 @@ class CSV extends Destination public function __construct( Device $deviceForExports, string $resourceId, - array $allowedColumns = [] + array $allowedColumns = [], + private readonly string $delimiter = ',', + private readonly string $enclosure = '"', + private readonly string $escape = '\\', + private readonly bool $includeHeaders = true, ) { $this->deviceForMigrations = $deviceForExports; $this->resourceId = $resourceId; @@ -86,9 +90,12 @@ protected function import(array $resources, callable $callback): void } } - $content = \implode('', $buffer['lines']); - if (\fwrite($handle, $content) === false) { - throw new \Exception("Failed to write to file: $log"); + foreach ($buffer['lines'] as $line) { + if ($line['type'] === 'csv') { + if (\fputcsv($handle, $line['data'], $this->delimiter, $this->enclosure, $this->escape) === false) { + throw new \Exception("Failed to write CSV line to file: $log"); + } + } } $buffer = [ @@ -114,16 +121,16 @@ protected function import(array $resources, callable $callback): void $csvData = $this->resourceToCSVData($resource); // Write headers if this is the first row of the file - if (!isset($csvHeader)) { - $headers = $this->toCSV(\array_keys($csvData)); - $buffer['lines'][] = $headers; - $buffer['size'] += \strlen($headers); + if (!isset($csvHeader) && $this->includeHeaders) { + $headers = \array_keys($csvData); + $buffer['lines'][] = ['type' => 'csv', 'data' => $headers]; + $buffer['size'] += \strlen(\implode($this->delimiter, $headers)) + 2; // Approximate size $csvHeader = true; } - $dataLine = $this->toCSV(\array_values($csvData)); - $buffer['lines'][] = $dataLine; - $buffer['size'] += \strlen($dataLine); + $dataValues = \array_values($csvData); + $buffer['lines'][] = ['type' => 'csv', 'data' => $dataValues]; + $buffer['size'] += \strlen(\implode($this->delimiter, $dataValues)) + 2; // Approximate size if ($buffer['size'] >= $bufferBytes) { $flushBuffer(); @@ -281,29 +288,4 @@ protected function convertObjectToCSV($value): string return \json_encode($value); } - /** - * Convert array to CSV line with proper escaping - * Uses standard CSV format with double-quote escaping - */ - protected function toCSV(array $array): string - { - $output = []; - foreach ($array as $value) { - $output[] = $this->escapeForCSV($value); - } - return \implode(',', $output) . "\n"; - } - - /** - * Escape a single value for CSV format - */ - protected function escapeForCSV(string $value): string - { - if (\strpbrk($value, ",\n\r\"") !== false) { - // Escape quotes by doubling them (CSV standard) - $escaped = \str_replace('"', '""', $value); - return '"' . $escaped . '"'; - } - return $value; - } } diff --git a/src/Migration/Resources/Database/Row.php b/src/Migration/Resources/Database/Row.php index 6b7e9ada..42d5bfca 100644 --- a/src/Migration/Resources/Database/Row.php +++ b/src/Migration/Resources/Database/Row.php @@ -18,8 +18,7 @@ public function __construct( private readonly Table $table, private readonly array $data = [], array $permissions = [] - ) - { + ) { $this->id = $id; $this->permissions = $permissions; } diff --git a/tests/Migration/Unit/General/CSVTest.php b/tests/Migration/Unit/General/CSVTest.php index 3414e5fb..549cef9c 100644 --- a/tests/Migration/Unit/General/CSVTest.php +++ b/tests/Migration/Unit/General/CSVTest.php @@ -3,11 +3,11 @@ namespace Migration\Unit\General; use PHPUnit\Framework\TestCase; -use Utopia\Migration\Sources\CSV; use Utopia\Migration\Destinations\CSV as DestinationCSV; +use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Row; use Utopia\Migration\Resources\Database\Table; -use Utopia\Migration\Resources\Database\Database; +use Utopia\Migration\Sources\CSV; use Utopia\Storage\Device\Local; /** @@ -19,12 +19,12 @@ public function testableImport(array $resources, callable $callback): void { $this->import($resources, $callback); } - + public function getLocalRoot(): string { return $this->local->getRoot(); } - + // Override shutdown to avoid transfer for testing public function shutdown(): void { @@ -79,85 +79,86 @@ public function testCSVExportBasic() $tempDir = sys_get_temp_dir() . '/csv_test_' . uniqid(); mkdir($tempDir, 0755, true); $exportDevice = new Local($tempDir); - + // Create CSV destination $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); - + // Create test data $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); - + $row1 = new Row('row1', $table, [ 'name' => 'John Doe', 'age' => 30, 'email' => 'john@example.com' ]); $row1->setPermissions(['read' => ['user:123']]); - + $row2 = new Row('row2', $table, [ 'name' => 'Jane Smith', 'age' => 25, 'email' => 'jane@example.com' ]); $row2->setPermissions(['read' => ['user:456']]); - + // Export the data - $csvDestination->testableImport([$row1, $row2], function($resources) { + $csvDestination->testableImport([$row1, $row2], function ($resources) { // Callback - verify resources are marked as successful foreach ($resources as $resource) { $this->assertEquals('success', $resource->getStatus()); } }); - + $csvDestination->shutdown(); - + // Verify CSV file was created in local temp directory - $expectedFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + // Note: The filename gets sanitized, so ':' becomes '_' + $expectedFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; $this->assertFileExists($expectedFile, 'CSV file should exist'); - + // Use proper CSV parsing $handle = fopen($expectedFile, 'r'); $this->assertNotFalse($handle); - + $header = fgetcsv($handle, 0, ',', '"', '"'); $row1Data = fgetcsv($handle, 0, ',', '"', '"'); $row2Data = fgetcsv($handle, 0, ',', '"', '"'); fclose($handle); - + $this->assertNotFalse($header); $this->assertNotFalse($row1Data); $this->assertNotFalse($row2Data); - + // Check header $this->assertContains('$id', $header); $this->assertContains('$permissions', $header); $this->assertContains('name', $header); $this->assertContains('age', $header); $this->assertContains('email', $header); - + // Check first row data $this->assertEquals('row1', $row1Data[0]); // $id $this->assertStringContainsString('user:123', $row1Data[1]); // $permissions $this->assertEquals('John Doe', $row1Data[2]); // name $this->assertEquals('30', $row1Data[3]); // age $this->assertEquals('john@example.com', $row1Data[4]); // email - + // Cleanup if (is_dir($tempDir)) { $this->recursiveDelete($tempDir); } } - + public function testCSVExportWithSpecialCharacters() { $tempDir = sys_get_temp_dir() . '/csv_test_special_' . uniqid(); $exportDevice = new Local($tempDir); - + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); - + $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); - + // Test data with special characters that need escaping $row = new Row('special_row', $table, [ 'quote_field' => 'Text with "quotes"', @@ -165,90 +166,90 @@ public function testCSVExportWithSpecialCharacters() 'newline_field' => "Text with\nnewlines", 'mixed_field' => 'Text with "quotes", commas, and\nnewlines' ]); - - $csvDestination->testableImport([$row], function($resources) {}); + + $csvDestination->testableImport([$row], function ($resources) {}); $csvDestination->shutdown(); - - $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; - + + $csvFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; + // Use proper CSV parsing $handle = fopen($csvFile, 'r'); $this->assertNotFalse($handle); - + $header = fgetcsv($handle, 0, ',', '"', '"'); $rowData = fgetcsv($handle, 0, ',', '"', '"'); fclose($handle); - + $this->assertNotFalse($header); $this->assertNotFalse($rowData); - + // Verify special characters are properly handled (fgetcsv($handle, 0, ',', '"', '"'); - $this->assertEquals('Text with "quotes"', $rowData[2]); // quote_field + $this->assertEquals('Text with "quotes"', $rowData[2]); // quote_field $this->assertEquals('Text, with, commas', $rowData[3]); // comma_field $this->assertEquals("Text with\nnewlines", $rowData[4]); // newline_field $this->assertEquals('Text with "quotes", commas, and\nnewlines', $rowData[5]); // mixed_field - + // Cleanup if (is_dir($tempDir)) { $this->recursiveDelete($tempDir); } } - + public function testCSVExportWithArrays() { $tempDir = sys_get_temp_dir() . '/csv_test_arrays_' . uniqid(); $exportDevice = new Local($tempDir); - + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); - + $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); - + $row = new Row('array_row', $table, [ 'tags' => ['php', 'csv', 'export'], 'metadata' => ['key1' => 'value1', 'key2' => 'value2'], 'empty_array' => [], 'nested' => [['id' => 1], ['id' => 2]] ]); - - $csvDestination->testableImport([$row], function($resources) {}); + + $csvDestination->testableImport([$row], function ($resources) {}); $csvDestination->shutdown(); - - $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; - + + $csvFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; + // Use proper CSV parsing $handle = fopen($csvFile, 'r'); $this->assertNotFalse($handle); - + $header = fgetcsv($handle, 0, ',', '"', '"'); $rowData = fgetcsv($handle, 0, ',', '"', '"'); fclose($handle); - + $this->assertNotFalse($header); $this->assertNotFalse($rowData); - + // Arrays should be JSON encoded $this->assertEquals('["php","csv","export"]', $rowData[2]); // tags $this->assertJson($rowData[3]); // metadata should be valid JSON $this->assertEquals('', $rowData[4]); // empty_array $this->assertJson($rowData[5]); // nested should be valid JSON - + // Cleanup if (is_dir($tempDir)) { $this->recursiveDelete($tempDir); } } - + public function testCSVExportWithNullValues() { $tempDir = sys_get_temp_dir() . '/csv_test_nulls_' . uniqid(); $exportDevice = new Local($tempDir); - + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); - + $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); - + $row = new Row('null_row', $table, [ 'name' => 'Test', 'null_field' => null, @@ -256,69 +257,69 @@ public function testCSVExportWithNullValues() 'zero' => 0, 'false_bool' => false ]); - - $csvDestination->testableImport([$row], function($resources) {}); + + $csvDestination->testableImport([$row], function ($resources) {}); $csvDestination->shutdown(); - - $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; - + + $csvFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; + // Use proper CSV parsing $handle = fopen($csvFile, 'r'); $this->assertNotFalse($handle); - + $header = fgetcsv($handle, 0, ',', '"', '"'); $rowData = fgetcsv($handle, 0, ',', '"', '"'); fclose($handle); - + $this->assertNotFalse($header); $this->assertNotFalse($rowData); - + $this->assertEquals('Test', $rowData[2]); // name $this->assertEquals('null', $rowData[3]); // null_field -> "null" string $this->assertEquals('', $rowData[4]); // empty_string $this->assertEquals('0', $rowData[5]); // zero $this->assertEquals('false', $rowData[6]); // false_bool - + // Cleanup if (is_dir($tempDir)) { $this->recursiveDelete($tempDir); } } - + public function testCSVExportWithAllowedAttributes() { $tempDir = sys_get_temp_dir() . '/csv_test_filtered_' . uniqid(); $exportDevice = new Local($tempDir); - + // Only allow specific attributes $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', ['name', 'email']); - + $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); - + $row = new Row('filtered_row', $table, [ 'name' => 'John Doe', 'age' => 30, 'email' => 'john@example.com', 'secret' => 'should_not_appear' ]); - - $csvDestination->testableImport([$row], function($resources) {}); + + $csvDestination->testableImport([$row], function ($resources) {}); $csvDestination->shutdown(); - - $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; - + + $csvFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; + // Use proper CSV parsing $handle = fopen($csvFile, 'r'); $this->assertNotFalse($handle); - + $header = fgetcsv($handle, 0, ',', '"', '"'); $rowData = fgetcsv($handle, 0, ',', '"', '"'); fclose($handle); - + $this->assertNotFalse($header); $this->assertNotFalse($rowData); - + // Should have $id, $permissions, and only allowed attributes $this->assertContains('$id', $header); $this->assertContains('$permissions', $header); @@ -326,24 +327,24 @@ public function testCSVExportWithAllowedAttributes() $this->assertContains('email', $header); $this->assertNotContains('age', $header); $this->assertNotContains('secret', $header); - + // Cleanup if (is_dir($tempDir)) { $this->recursiveDelete($tempDir); } } - + public function testCSVExportImportCompatibility() { $tempDir = sys_get_temp_dir() . '/csv_test_compat_' . uniqid(); $exportDevice = new Local($tempDir); - + // Export data $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); - + $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); - + $originalData = [ 'name' => 'John Doe', 'age' => 30, @@ -353,53 +354,53 @@ public function testCSVExportImportCompatibility() 'empty_field' => '', 'bool_field' => true ]; - + $row = new Row('compat_row', $table, $originalData); $row->setPermissions(['read' => ['user:123']]); - - $csvDestination->testableImport([$row], function($resources) {}); + + $csvDestination->testableImport([$row], function ($resources) {}); $csvDestination->shutdown(); - + // Verify the exported CSV can be parsed by PHP's built-in CSV functions - $csvFile = $csvDestination->getLocalRoot() . '/test_db:test_table_id.csv'; + $csvFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; $this->assertFileExists($csvFile); - + $handle = fopen($csvFile, 'r'); $this->assertNotFalse($handle); - + $header = fgetcsv($handle, 0, ',', '"', '"'); $data = fgetcsv($handle, 0, ',', '"', '"'); fclose($handle); - + $this->assertNotFalse($header); $this->assertNotFalse($data); - + // Verify we can reconstruct the data $reconstructed = \array_combine($header, $data); - + $this->assertEquals('compat_row', $reconstructed['$id']); $this->assertEquals('John Doe', $reconstructed['name']); $this->assertEquals('30', $reconstructed['age']); $this->assertEquals('null', $reconstructed['null_field']); // null becomes "null" string $this->assertEquals('', $reconstructed['empty_field']); $this->assertEquals('true', $reconstructed['bool_field']); // bool becomes string - + // Arrays should be valid JSON that can be decoded $this->assertJson($reconstructed['tags']); $this->assertJson($reconstructed['metadata']); - + $tagsArray = json_decode($reconstructed['tags'], true); $metadataArray = json_decode($reconstructed['metadata'], true); - + $this->assertEquals(['php', 'csv'], $tagsArray); $this->assertEquals(['key' => 'value'], $metadataArray); - + // Cleanup if (is_dir($tempDir)) { $this->recursiveDelete($tempDir); } } - + private function recursiveDelete(string $dir): void { if (is_dir($dir)) { From 8931241d175e4a2f996321c9af526d5ed1117424 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 16 Sep 2025 13:39:24 +1200 Subject: [PATCH 7/9] Remove redundant type --- src/Migration/Destinations/CSV.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index 76d560f6..797c912f 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -91,10 +91,8 @@ protected function import(array $resources, callable $callback): void } foreach ($buffer['lines'] as $line) { - if ($line['type'] === 'csv') { - if (\fputcsv($handle, $line['data'], $this->delimiter, $this->enclosure, $this->escape) === false) { - throw new \Exception("Failed to write CSV line to file: $log"); - } + if (\fputcsv($handle, $line, $this->delimiter, $this->enclosure, $this->escape) === false) { + throw new \Exception("Failed to write CSV line to file: $log"); } } @@ -123,13 +121,13 @@ protected function import(array $resources, callable $callback): void // Write headers if this is the first row of the file if (!isset($csvHeader) && $this->includeHeaders) { $headers = \array_keys($csvData); - $buffer['lines'][] = ['type' => 'csv', 'data' => $headers]; + $buffer['lines'][] = $headers; $buffer['size'] += \strlen(\implode($this->delimiter, $headers)) + 2; // Approximate size $csvHeader = true; } $dataValues = \array_values($csvData); - $buffer['lines'][] = ['type' => 'csv', 'data' => $dataValues]; + $buffer['lines'][] = $dataValues; $buffer['size'] += \strlen(\implode($this->delimiter, $dataValues)) + 2; // Approximate size if ($buffer['size'] >= $bufferBytes) { From 7a9c8be173637f1c4905a7c168779f34f55c8f79 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 21:48:50 +1200 Subject: [PATCH 8/9] Add dir + file name --- src/Migration/Destinations/CSV.php | 32 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Migration/Destinations/CSV.php b/src/Migration/Destinations/CSV.php index 797c912f..a61cc5bc 100644 --- a/src/Migration/Destinations/CSV.php +++ b/src/Migration/Destinations/CSV.php @@ -15,8 +15,9 @@ class CSV extends Destination { - protected Device $deviceForMigrations; + protected Device $deviceForFiles; protected string $resourceId; + protected string $directory; protected string $outputFile; protected Local $local; @@ -29,17 +30,20 @@ class CSV extends Destination * @throws \Exception */ public function __construct( - Device $deviceForExports, + Device $deviceForFiles, string $resourceId, + string $directory, + string $filename, array $allowedColumns = [], private readonly string $delimiter = ',', private readonly string $enclosure = '"', private readonly string $escape = '\\', private readonly bool $includeHeaders = true, ) { - $this->deviceForMigrations = $deviceForExports; + $this->deviceForFiles = $deviceForFiles; $this->resourceId = $resourceId; - $this->outputFile = $this->sanitizeFilename($resourceId); + $this->directory = $directory; + $this->outputFile = $this->sanitizeFilename($filename); $this->local = new Local(\sys_get_temp_dir() . '/csv_export_' . uniqid()); $this->local->setTransferChunkSize(Transfer::STORAGE_MAX_CHUNK_SIZE); $this->createDirectory($this->local->getRoot()); @@ -159,26 +163,26 @@ protected function import(array $resources, callable $callback): void public function shutdown(): void { $filename = $this->outputFile . '.csv'; - $sourceFilePath = $this->local->getPath($filename); - $destFilePath = $this->deviceForMigrations->getPath($filename); + $sourcePath = $this->local->getPath($filename); + $destPath = $this->deviceForFiles->getPath($this->directory . '/' . $filename); // Check if the CSV file was actually created - if (!$this->local->exists($sourceFilePath)) { + if (!$this->local->exists($sourcePath)) { throw new \Exception("No data to export for resource: $this->resourceId"); } try { // Transfer expects absolute paths within each device $result = $this->local->transfer( - $filename, - $filename, - $this->deviceForMigrations + $sourcePath, + $destPath, + $this->deviceForFiles ); if ($result === false) { - throw new \Exception('Error transferring to ' . $this->deviceForMigrations->getRoot() . '/' . $filename); + throw new \Exception('Error transferring to ' . $this->deviceForFiles->getRoot() . '/' . $filename); } - if (!$this->deviceForMigrations->exists($destFilePath)) { - throw new \Exception('File not found on destination: ' . $destFilePath); + if (!$this->deviceForFiles->exists($destPath)) { + throw new \Exception('File not found on destination: ' . $destPath); } } finally { // Clean up the temporary directory @@ -221,6 +225,8 @@ protected function resourceToCSVData(Row $resource): array $data = [ '$id' => $resource->getId(), '$permissions' => $resource->getPermissions(), + '$createdAt' => $resource->getCreatedAt(), + '$updatedAt' => $resource->getUpdatedAt(), ]; // Add all attributes if no filter specified, otherwise only allowed ones From c46083a885cf86ad80b1ee71be88d99496906efa Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 24 Sep 2025 22:25:55 +1200 Subject: [PATCH 9/9] Update tests --- tests/Migration/Unit/General/CSVTest.php | 60 ++++++++++++++---------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/tests/Migration/Unit/General/CSVTest.php b/tests/Migration/Unit/General/CSVTest.php index 549cef9c..d98ab0ac 100644 --- a/tests/Migration/Unit/General/CSVTest.php +++ b/tests/Migration/Unit/General/CSVTest.php @@ -81,7 +81,7 @@ public function testCSVExportBasic() $exportDevice = new Local($tempDir); // Create CSV destination - $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', '', 'test_db_test_table_id'); // Create test data $database = new Database('test_db'); @@ -112,7 +112,6 @@ public function testCSVExportBasic() $csvDestination->shutdown(); // Verify CSV file was created in local temp directory - // Note: The filename gets sanitized, so ':' becomes '_' $expectedFile = $csvDestination->getLocalRoot() . '/test_db_test_table_id.csv'; $this->assertFileExists($expectedFile, 'CSV file should exist'); @@ -132,6 +131,8 @@ public function testCSVExportBasic() // Check header $this->assertContains('$id', $header); $this->assertContains('$permissions', $header); + $this->assertContains('$createdAt', $header); + $this->assertContains('$updatedAt', $header); $this->assertContains('name', $header); $this->assertContains('age', $header); $this->assertContains('email', $header); @@ -139,9 +140,10 @@ public function testCSVExportBasic() // Check first row data $this->assertEquals('row1', $row1Data[0]); // $id $this->assertStringContainsString('user:123', $row1Data[1]); // $permissions - $this->assertEquals('John Doe', $row1Data[2]); // name - $this->assertEquals('30', $row1Data[3]); // age - $this->assertEquals('john@example.com', $row1Data[4]); // email + // $createdAt and $updatedAt are empty for test data + $this->assertEquals('John Doe', $row1Data[4]); // name + $this->assertEquals('30', $row1Data[5]); // age + $this->assertEquals('john@example.com', $row1Data[6]); // email // Cleanup if (is_dir($tempDir)) { @@ -154,7 +156,7 @@ public function testCSVExportWithSpecialCharacters() $tempDir = sys_get_temp_dir() . '/csv_test_special_' . uniqid(); $exportDevice = new Local($tempDir); - $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', '', 'test_db_test_table_id'); $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); @@ -183,11 +185,12 @@ public function testCSVExportWithSpecialCharacters() $this->assertNotFalse($header); $this->assertNotFalse($rowData); - // Verify special characters are properly handled (fgetcsv($handle, 0, ',', '"', '"'); - $this->assertEquals('Text with "quotes"', $rowData[2]); // quote_field - $this->assertEquals('Text, with, commas', $rowData[3]); // comma_field - $this->assertEquals("Text with\nnewlines", $rowData[4]); // newline_field - $this->assertEquals('Text with "quotes", commas, and\nnewlines', $rowData[5]); // mixed_field + // Verify special characters are properly handled + // Indices are shifted by 2 due to $createdAt and $updatedAt + $this->assertEquals('Text with "quotes"', $rowData[4]); // quote_field + $this->assertEquals('Text, with, commas', $rowData[5]); // comma_field + $this->assertEquals("Text with\nnewlines", $rowData[6]); // newline_field + $this->assertEquals('Text with "quotes", commas, and\nnewlines', $rowData[7]); // mixed_field // Cleanup if (is_dir($tempDir)) { @@ -200,7 +203,7 @@ public function testCSVExportWithArrays() $tempDir = sys_get_temp_dir() . '/csv_test_arrays_' . uniqid(); $exportDevice = new Local($tempDir); - $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', '', 'test_db_test_table_id'); $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); @@ -229,10 +232,11 @@ public function testCSVExportWithArrays() $this->assertNotFalse($rowData); // Arrays should be JSON encoded - $this->assertEquals('["php","csv","export"]', $rowData[2]); // tags - $this->assertJson($rowData[3]); // metadata should be valid JSON - $this->assertEquals('', $rowData[4]); // empty_array - $this->assertJson($rowData[5]); // nested should be valid JSON + // Indices are shifted by 2 due to $createdAt and $updatedAt + $this->assertEquals('["php","csv","export"]', $rowData[4]); // tags + $this->assertJson($rowData[5]); // metadata should be valid JSON + $this->assertEquals('', $rowData[6]); // empty_array + $this->assertJson($rowData[7]); // nested should be valid JSON // Cleanup if (is_dir($tempDir)) { @@ -245,7 +249,7 @@ public function testCSVExportWithNullValues() $tempDir = sys_get_temp_dir() . '/csv_test_nulls_' . uniqid(); $exportDevice = new Local($tempDir); - $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', '', 'test_db_test_table_id'); $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); @@ -274,11 +278,12 @@ public function testCSVExportWithNullValues() $this->assertNotFalse($header); $this->assertNotFalse($rowData); - $this->assertEquals('Test', $rowData[2]); // name - $this->assertEquals('null', $rowData[3]); // null_field -> "null" string - $this->assertEquals('', $rowData[4]); // empty_string - $this->assertEquals('0', $rowData[5]); // zero - $this->assertEquals('false', $rowData[6]); // false_bool + // Indices are shifted by 2 due to $createdAt and $updatedAt + $this->assertEquals('Test', $rowData[4]); // name + $this->assertEquals('null', $rowData[5]); // null_field -> "null" string + $this->assertEquals('', $rowData[6]); // empty_string + $this->assertEquals('0', $rowData[7]); // zero + $this->assertEquals('false', $rowData[8]); // false_bool // Cleanup if (is_dir($tempDir)) { @@ -292,7 +297,7 @@ public function testCSVExportWithAllowedAttributes() $exportDevice = new Local($tempDir); // Only allow specific attributes - $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', ['name', 'email']); + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', '', 'test_db_test_table_id', ['name', 'email']); $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); @@ -320,9 +325,11 @@ public function testCSVExportWithAllowedAttributes() $this->assertNotFalse($header); $this->assertNotFalse($rowData); - // Should have $id, $permissions, and only allowed attributes + // Should have $id, $permissions, $createdAt, $updatedAt, and only allowed attributes $this->assertContains('$id', $header); $this->assertContains('$permissions', $header); + $this->assertContains('$createdAt', $header); + $this->assertContains('$updatedAt', $header); $this->assertContains('name', $header); $this->assertContains('email', $header); $this->assertNotContains('age', $header); @@ -340,7 +347,7 @@ public function testCSVExportImportCompatibility() $exportDevice = new Local($tempDir); // Export data - $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id'); + $csvDestination = new TestCSV($exportDevice, 'test_db:test_table_id', '', 'test_db_test_table_id'); $database = new Database('test_db'); $table = new Table($database, 'test_table', 'test_table_id'); @@ -384,6 +391,9 @@ public function testCSVExportImportCompatibility() $this->assertEquals('null', $reconstructed['null_field']); // null becomes "null" string $this->assertEquals('', $reconstructed['empty_field']); $this->assertEquals('true', $reconstructed['bool_field']); // bool becomes string + // Check that createdAt and updatedAt are in the reconstructed data + $this->assertArrayHasKey('$createdAt', $reconstructed); + $this->assertArrayHasKey('$updatedAt', $reconstructed); // Arrays should be valid JSON that can be decoded $this->assertJson($reconstructed['tags']);