From 9eedfc0df856dfa6487ec53bdcfed3041b6e5528 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:23:17 +0400 Subject: [PATCH 1/9] feat: support out-of-order chunked uploads in Local adapter - Remove fragile _chunks.log file and use .part.{n} files as single source of truth - Fix duplicate chunk detection in uploadData() to match upload() - Write part files before counting to avoid count/desync issues - Add countChunks() helper using glob() for reliable chunk counting - Add out-of-order upload tests for Local and S3 adapters - Add .gitignore to disk-a test resources to ignore test artifacts --- src/Storage/Device/Local.php | 65 +++++++++++++----------------- tests/Storage/Device/LocalTest.php | 60 +++++++++++++++++++++------ tests/Storage/S3Base.php | 47 +++++++++++++++++++++ tests/resources/disk-a/.gitignore | 8 ++++ 4 files changed, 131 insertions(+), 49 deletions(-) create mode 100644 tests/resources/disk-a/.gitignore diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 8236129e..0d1bd5bf 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -66,29 +66,20 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks return $chunks; } - $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - $this->createDirectory(\dirname($tmp)); + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path); + $this->createDirectory($tmp); - $chunkFilePath = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk; + $chunkFilePath = $tmp.DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk; // skip writing chunk if the chunk was re-uploaded if (! file_exists($chunkFilePath)) { - if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { - throw new Exception('Can\'t write chunk log '.$tmp); + if (! \rename($source, $chunkFilePath)) { + throw new Exception('Failed to write chunk '.$chunk); } } - $chunkLogs = file($tmp); - if (! $chunkLogs) { - throw new Exception('Unable to read chunk log '.$tmp); - } - - $chunksReceived = count(file($tmp)); - - if (! \rename($source, $chunkFilePath)) { - throw new Exception('Failed to write chunk '.$chunk); - } + $chunksReceived = $this->countChunks($tmp, $path); if ($chunks === $chunksReceived) { $this->joinChunks($path, $chunks); @@ -122,24 +113,21 @@ public function uploadData(string $data, string $path, string $contentType, int return $chunks; } - $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - $this->createDirectory(\dirname($tmp)); - if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { - throw new Exception('Can\'t write chunk log '.$tmp); - } + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path); + $this->createDirectory($tmp); - $chunkLogs = file($tmp); - if (! $chunkLogs) { - throw new Exception('Unable to read chunk log '.$tmp); - } - - $chunksReceived = count(file($tmp)); + $chunkFilePath = $tmp.DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk; - if (! \file_put_contents(dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk, $data)) { - throw new Exception('Failed to write chunk '.$chunk); + // skip writing chunk if the chunk was re-uploaded + if (! file_exists($chunkFilePath)) { + if (! \file_put_contents($chunkFilePath, $data)) { + throw new Exception('Failed to write chunk '.$chunk); + } } + $chunksReceived = $this->countChunks($tmp, $path); + if ($chunks === $chunksReceived) { $this->joinChunks($path, $chunks); @@ -149,10 +137,17 @@ public function uploadData(string $data, string $path, string $contentType, int return $chunksReceived; } + private function countChunks(string $tmp, string $path): int + { + $pattern = $tmp.DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.*'; + $files = \glob($pattern); + + return $files === false ? 0 : \count($files); + } + private function joinChunks(string $path, int $chunks): void { - $tmpDir = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path); - $tmp = $tmpDir.DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path); $tmpAssemble = \dirname($path).DIRECTORY_SEPARATOR.'tmp_assemble_'.\basename($path); $dest = \fopen($tmpAssemble, 'wb'); @@ -162,7 +157,7 @@ private function joinChunks(string $path, int $chunks): void $partsToUnlink = []; for ($i = 1; $i <= $chunks; $i++) { - $part = $tmpDir.DIRECTORY_SEPARATOR.\pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; + $part = $tmp.DIRECTORY_SEPARATOR.\pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; $src = @\fopen($part, 'rb'); if ($src === false) { \fclose($dest); @@ -193,12 +188,8 @@ private function joinChunks(string $path, int $chunks): void } } - if (! \unlink($tmp)) { - \trigger_error('Failed to remove chunk log '.$tmp, E_USER_WARNING); - } - - if (! \rmdir($tmpDir)) { - \trigger_error('Failed to remove temporary chunk directory '.$tmpDir, E_USER_WARNING); + if (! \rmdir($tmp)) { + \trigger_error('Failed to remove temporary chunk directory '.$tmp, E_USER_WARNING); } } diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 5d1f0f9a..2146d413 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -2,7 +2,6 @@ namespace Utopia\Tests\Storage\Device; -use Exception; use PHPUnit\Framework\TestCase; use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Local; @@ -483,7 +482,7 @@ public function testJoinChunksCleansUpTempFilesOnSuccess(): void $storage->delete($storage->getRoot(), true); } - public function testJoinChunksMissingPartThrowsAndPreservesState(): void + public function testJoinChunksMissingPartDoesNotFinalize(): void { $storage = $this->makeJoinTestStorage(); $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'test.dat'; @@ -497,17 +496,12 @@ public function testJoinChunksMissingPartThrowsAndPreservesState(): void // final upload triggers assembly. \unlink($tmpDir.DIRECTORY_SEPARATOR.'test.part.1'); - $exceptionThrown = false; - try { - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); - } catch (Exception $e) { - $exceptionThrown = true; - $this->assertStringContainsString('Failed to open chunk', $e->getMessage()); - } + // Uploading the final chunk should NOT throw or finalize, + // because part 1 is missing and the part-file count is only 2. + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); - $this->assertTrue($exceptionThrown, 'Exception should be thrown when a chunk is missing'); - $this->assertFalse(\file_exists($dest), 'Final file must not be created on assembly failure'); - $this->assertFalse(\file_exists($tmpAssemble), 'Temp assembly file must be cleaned up on failure'); + $this->assertFalse(\file_exists($dest), 'Final file must not be created when a chunk is missing'); + $this->assertFalse(\file_exists($tmpAssemble), 'Temp assembly file must not be created'); // Surviving parts must remain so the upload can be retried. $this->assertTrue( \file_exists($tmpDir.DIRECTORY_SEPARATOR.'test.part.2'), @@ -518,6 +512,11 @@ public function testJoinChunksMissingPartThrowsAndPreservesState(): void 'Part 3 must be preserved for retry' ); + // Re-upload the missing chunk — assembly should now succeed. + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $this->assertTrue(\file_exists($dest), 'Final file should be created after missing chunk is re-uploaded'); + $this->assertSame('AAAABBBBCCCC', \file_get_contents($dest), 'Re-uploaded chunk must allow correct assembly'); + $storage->delete($storage->getRoot(), true); } @@ -541,4 +540,41 @@ public function testJoinChunksStaleAssemblyFileIsOverwritten(): void $storage->delete($storage->getRoot(), true); } + + public function testOutOfOrderUpload(): void + { + $storage = $this->makeJoinTestStorage(); + $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'out-of-order.dat'; + + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $this->assertFalse(\file_exists($dest), 'File should not be assembled after chunk 3'); + + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $this->assertFalse(\file_exists($dest), 'File should not be assembled after chunk 1'); + + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $this->assertTrue(\file_exists($dest), 'File should be assembled after final chunk'); + $this->assertSame('AAAABBBBCCCC', \file_get_contents($dest), 'Chunks must be assembled in correct order'); + + $storage->delete($storage->getRoot(), true); + } + + public function testOutOfOrderUploadWithRetry(): void + { + $storage = $this->makeJoinTestStorage(); + $dest = $storage->getRoot().DIRECTORY_SEPARATOR.'out-of-order-retry.dat'; + + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + + // Re-upload chunk 2 (duplicate) — should be silently ignored + $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $this->assertFalse(\file_exists($dest), 'File should not be assembled after duplicate retry'); + + $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $this->assertTrue(\file_exists($dest), 'File should be assembled after final chunk'); + $this->assertSame('AAAABBBBCCCC', \file_get_contents($dest), 'Duplicate retry must not corrupt final file'); + + $storage->delete($storage->getRoot(), true); + } } diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 5b428606..3938aa76 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -364,6 +364,53 @@ public function testPartUploadRetry() return $dest; } + public function testOutOfOrderPartUpload() + { + $source = __DIR__.'/../resources/disk-a/large_file.mp4'; + $dest = $this->object->getPath('uploaded-out-of-order.mp4'); + $totalSize = \filesize($source); + // AWS S3 requires each part to be at least 5MB except for last part + $chunkSize = 5 * 1024 * 1024; + + $chunks = ceil($totalSize / $chunkSize); + + // Read all chunk contents into memory so we can upload out of order + $parts = []; + $handle = @fopen($source, 'rb'); + $chunkNum = 1; + while ($chunkNum <= $chunks) { + $contents = fread($handle, $chunkSize); + $parts[$chunkNum] = $contents; + $chunkNum++; + } + @fclose($handle); + + $metadata = [ + 'parts' => [], + 'chunks' => 0, + 'uploadId' => null, + 'content_type' => \mime_content_type($source), + ]; + + // Upload chunks in reverse order + for ($i = $chunks; $i >= 1; $i--) { + $op = __DIR__.'/chunk.part'; + $cc = fopen($op, 'wb'); + fwrite($cc, $parts[$i]); + fclose($cc); + $this->object->upload($op, $dest, $i, $chunks, $metadata); + unlink($op); + } + + $this->assertEquals(\filesize($source), $this->object->getFileSize($dest)); + + // S3 doesnt provide a method to get a proper MD5-hash of a file created using multipart upload + // TODO + // $this->assertEquals(\md5_file($source), $this->object->getFileHash($dest)); + // $this->object->delete($dest); + return $dest; + } + /** * @depends testPartUpload */ diff --git a/tests/resources/disk-a/.gitignore b/tests/resources/disk-a/.gitignore new file mode 100644 index 00000000..fceac3a5 --- /dev/null +++ b/tests/resources/disk-a/.gitignore @@ -0,0 +1,8 @@ +* +!.gitignore +!config.xml +!kitten-1.jpg +!kitten-2.jpg +!kitten-3.gif +!large_file.mp4 +!lorem.txt From 4f40733d24232dd5f8092a1a2926908d49411e4b Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:33:28 +0400 Subject: [PATCH 2/9] fix: clean up duplicate chunk sources and validate part suffixes in countChunks --- src/Storage/Device/Local.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 0d1bd5bf..59f44abf 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -77,6 +77,10 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks if (! \rename($source, $chunkFilePath)) { throw new Exception('Failed to write chunk '.$chunk); } + } else { + if (\file_exists($source)) { + \unlink($source); + } } $chunksReceived = $this->countChunks($tmp, $path); @@ -141,8 +145,18 @@ private function countChunks(string $tmp, string $path): int { $pattern = $tmp.DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.*'; $files = \glob($pattern); + if ($files === false) { + return 0; + } + + $count = 0; + foreach ($files as $file) { + if (\preg_match('/\.part\.\d+$/', $file)) { + $count++; + } + } - return $files === false ? 0 : \count($files); + return $count; } private function joinChunks(string $path, int $chunks): void From 64fab0004e6c1671a87a06ca52c414890c25bdd8 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:37:22 +0400 Subject: [PATCH 3/9] fix: use composer install and php 8.3 in Dockerfile to prevent dependency drift --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e9659f6..e5a5c91e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,14 +8,14 @@ WORKDIR /usr/local/src/ COPY composer.lock /usr/local/src/ COPY composer.json /usr/local/src/ -RUN composer update \ +RUN composer install \ --ignore-platform-reqs \ --optimize-autoloader \ --no-plugins \ --no-scripts \ --prefer-dist -FROM php:8.1-cli-alpine as compile +FROM php:8.3-cli-alpine as compile ENV PHP_ZSTD_VERSION="master" ENV PHP_BROTLI_VERSION="7ae4fcd8b81a65d7521c298cae49af386d1ea4e3" From a5c84576ee6469e4f2355fdaaa5b73be77ab68f3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:41:44 +0400 Subject: [PATCH 4/9] Update Dockerfile stages and composer.lock --- Dockerfile | 10 ++-- composer.lock | 145 +++++++++++++++++++++++++------------------------- 2 files changed, 78 insertions(+), 77 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5a5c91e..0bf6ba47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM composer:2.0 as composer +FROM composer:2.0 AS composer ARG TESTING=false ENV TESTING=$TESTING @@ -15,7 +15,7 @@ RUN composer install \ --no-scripts \ --prefer-dist -FROM php:8.3-cli-alpine as compile +FROM php:8.1-cli-alpine AS compile ENV PHP_ZSTD_VERSION="master" ENV PHP_BROTLI_VERSION="7ae4fcd8b81a65d7521c298cae49af386d1ea4e3" @@ -42,7 +42,7 @@ RUN git clone --recursive --depth 1 --branch $PHP_ZSTD_VERSION https://github.co && make && make install ## Brotli Extension -FROM compile as brotli +FROM compile AS brotli RUN git clone https://github.com/kjdev/php-ext-brotli.git \ && cd php-ext-brotli \ && git reset --hard $PHP_BROTLI_VERSION \ @@ -69,7 +69,7 @@ RUN git clone --recursive https://github.com/kjdev/php-ext-snappy.git \ && make && make install ## Xz Extension -FROM compile as xz +FROM compile AS xz RUN wget https://tukaani.org/xz/xz-${PHP_XZ_VERSION}.tar.xz -O xz.tar.xz \ && tar -xJf xz.tar.xz \ && rm xz.tar.xz \ @@ -87,7 +87,7 @@ RUN git clone https://github.com/codemasher/php-ext-xz.git --branch ${PHP_EXT_XZ && ./configure \ && make && make install -FROM compile as final +FROM compile AS final LABEL maintainer="team@appwrite.io" diff --git a/composer.lock b/composer.lock index a79e6646..ca03a505 100644 --- a/composer.lock +++ b/composer.lock @@ -145,23 +145,23 @@ }, { "name": "google/protobuf", - "version": "v4.33.5", + "version": "v4.33.6", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "ebe8010a61b2ae0cff0d246fe1c4d44e9f7dfa6d" + "reference": "84b008c23915ed94536737eae46f41ba3bccfe67" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/ebe8010a61b2ae0cff0d246fe1c4d44e9f7dfa6d", - "reference": "ebe8010a61b2ae0cff0d246fe1c4d44e9f7dfa6d", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/84b008c23915ed94536737eae46f41ba3bccfe67", + "reference": "84b008c23915ed94536737eae46f41ba3bccfe67", "shasum": "" }, "require": { "php": ">=8.1.0" }, "require-dev": { - "phpunit/phpunit": ">=5.0.0 <8.5.27" + "phpunit/phpunit": ">=10.5.62 <11.0.0" }, "suggest": { "ext-bcmath": "Need to support JSON deserialization" @@ -183,9 +183,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.5" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.6" }, - "time": "2026-01-29T20:49:00+00:00" + "time": "2026-03-18T17:32:05+00:00" }, { "name": "nyholm/psr7", @@ -333,16 +333,16 @@ }, { "name": "open-telemetry/api", - "version": "1.8.0", + "version": "1.9.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "df5197c6fd0ddd8e9883b87de042d9341300e2ad" + "reference": "6f8d237ce2c304ca85f31970f788e7f074d147be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/df5197c6fd0ddd8e9883b87de042d9341300e2ad", - "reference": "df5197c6fd0ddd8e9883b87de042d9341300e2ad", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/6f8d237ce2c304ca85f31970f788e7f074d147be", + "reference": "6f8d237ce2c304ca85f31970f788e7f074d147be", "shasum": "" }, "require": { @@ -399,20 +399,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2026-01-21T04:14:03+00:00" + "time": "2026-02-25T13:24:05+00:00" }, { "name": "open-telemetry/context", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/context.git", - "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf" + "reference": "3c414b246e0dabb7d6145404e6a5e4536ca18d07" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/d4c4470b541ce72000d18c339cfee633e4c8e0cf", - "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf", + "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/3c414b246e0dabb7d6145404e6a5e4536ca18d07", + "reference": "3c414b246e0dabb7d6145404e6a5e4536ca18d07", "shasum": "" }, "require": { @@ -454,11 +454,11 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/php", + "docs": "https://opentelemetry.io/docs/languages/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-09-19T00:05:49+00:00" + "time": "2025-10-19T06:44:33+00:00" }, { "name": "open-telemetry/exporter-otlp", @@ -526,16 +526,16 @@ }, { "name": "open-telemetry/gen-otlp-protobuf", - "version": "1.8.0", + "version": "1.9.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", - "reference": "673af5b06545b513466081884b47ef15a536edde" + "reference": "a229cf161d42001d64c8f21e8f678581fe1c66b9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/673af5b06545b513466081884b47ef15a536edde", - "reference": "673af5b06545b513466081884b47ef15a536edde", + "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/a229cf161d42001d64c8f21e8f678581fe1c66b9", + "reference": "a229cf161d42001d64c8f21e8f678581fe1c66b9", "shasum": "" }, "require": { @@ -581,30 +581,30 @@ ], "support": { "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", - "docs": "https://opentelemetry.io/docs/php", + "docs": "https://opentelemetry.io/docs/languages/php", "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-09-17T23:10:12+00:00" + "time": "2025-10-19T06:44:33+00:00" }, { "name": "open-telemetry/sdk", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "c76f91203bf7ef98ab3f4e0a82ca21699af185e1" + "reference": "6e3d0ce93e76555dd5e2f1d19443ff45b990e410" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/c76f91203bf7ef98ab3f4e0a82ca21699af185e1", - "reference": "c76f91203bf7ef98ab3f4e0a82ca21699af185e1", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/6e3d0ce93e76555dd5e2f1d19443ff45b990e410", + "reference": "6e3d0ce93e76555dd5e2f1d19443ff45b990e410", "shasum": "" }, "require": { "ext-json": "*", "nyholm/psr7-server": "^1.1", - "open-telemetry/api": "^1.7", + "open-telemetry/api": "^1.8", "open-telemetry/context": "^1.4", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", @@ -682,7 +682,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2026-01-28T11:38:11+00:00" + "time": "2026-03-21T11:50:01+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1306,16 +1306,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.5", + "version": "v7.4.8", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "84bb634857a893cc146cceb467e31b3f02c5fe9f" + "reference": "01933e626c3de76bea1e22641e205e78f6a34342" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/84bb634857a893cc146cceb467e31b3f02c5fe9f", - "reference": "84bb634857a893cc146cceb467e31b3f02c5fe9f", + "url": "https://api.github.com/repos/symfony/http-client/zipball/01933e626c3de76bea1e22641e205e78f6a34342", + "reference": "01933e626c3de76bea1e22641e205e78f6a34342", "shasum": "" }, "require": { @@ -1383,7 +1383,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.5" + "source": "https://github.com/symfony/http-client/tree/v7.4.8" }, "funding": [ { @@ -1403,7 +1403,7 @@ "type": "tidelift" } ], - "time": "2026-01-27T16:16:02+00:00" + "time": "2026-03-30T12:55:43+00:00" }, { "name": "symfony/http-client-contracts", @@ -1485,16 +1485,16 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "v1.33.0", + "version": "v1.37.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "6d857f4d76bd4b343eac26d6b539585d2bc56493" + "reference": "6a21eb99c6973357967f6ce3708cd55a6bec6315" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/6d857f4d76bd4b343eac26d6b539585d2bc56493", - "reference": "6d857f4d76bd4b343eac26d6b539585d2bc56493", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/6a21eb99c6973357967f6ce3708cd55a6bec6315", + "reference": "6a21eb99c6973357967f6ce3708cd55a6bec6315", "shasum": "" }, "require": { @@ -1546,7 +1546,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.33.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.37.0" }, "funding": [ { @@ -1566,20 +1566,20 @@ "type": "tidelift" } ], - "time": "2024-12-23T08:48:59+00:00" + "time": "2026-04-10T17:25:58+00:00" }, { "name": "symfony/polyfill-php82", - "version": "v1.33.0", + "version": "v1.37.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php82.git", - "reference": "5d2ed36f7734637dacc025f179698031951b1692" + "reference": "34808efe3e68f69685796f7c253a2f1d8ea9df59" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/5d2ed36f7734637dacc025f179698031951b1692", - "reference": "5d2ed36f7734637dacc025f179698031951b1692", + "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/34808efe3e68f69685796f7c253a2f1d8ea9df59", + "reference": "34808efe3e68f69685796f7c253a2f1d8ea9df59", "shasum": "" }, "require": { @@ -1626,7 +1626,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php82/tree/v1.33.0" + "source": "https://github.com/symfony/polyfill-php82/tree/v1.37.0" }, "funding": [ { @@ -1646,20 +1646,20 @@ "type": "tidelift" } ], - "time": "2024-09-09T11:45:10+00:00" + "time": "2026-04-10T16:19:22+00:00" }, { "name": "symfony/polyfill-php83", - "version": "v1.33.0", + "version": "v1.37.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php83.git", - "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5" + "reference": "3600c2cb22399e25bb226e4a135ce91eeb2a6149" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/17f6f9a6b1735c0f163024d959f700cfbc5155e5", - "reference": "17f6f9a6b1735c0f163024d959f700cfbc5155e5", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/3600c2cb22399e25bb226e4a135ce91eeb2a6149", + "reference": "3600c2cb22399e25bb226e4a135ce91eeb2a6149", "shasum": "" }, "require": { @@ -1706,7 +1706,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php83/tree/v1.33.0" + "source": "https://github.com/symfony/polyfill-php83/tree/v1.37.0" }, "funding": [ { @@ -1726,7 +1726,7 @@ "type": "tidelift" } ], - "time": "2025-07-08T02:45:35+00:00" + "time": "2026-04-10T17:25:58+00:00" }, { "name": "symfony/service-contracts", @@ -1869,16 +1869,16 @@ }, { "name": "utopia-php/system", - "version": "0.10.0", + "version": "0.10.1", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "6441a9c180958a373e5ddb330264dd638539dfdb" + "reference": "7c1669533bb9c285de19191270c8c1439161a78a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/6441a9c180958a373e5ddb330264dd638539dfdb", - "reference": "6441a9c180958a373e5ddb330264dd638539dfdb", + "url": "https://api.github.com/repos/utopia-php/system/zipball/7c1669533bb9c285de19191270c8c1439161a78a", + "reference": "7c1669533bb9c285de19191270c8c1439161a78a", "shasum": "" }, "require": { @@ -1919,9 +1919,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.10.0" + "source": "https://github.com/utopia-php/system/tree/0.10.1" }, - "time": "2025-10-15T19:12:00+00:00" + "time": "2026-03-15T21:07:41+00:00" }, { "name": "utopia-php/telemetry", @@ -2096,16 +2096,16 @@ }, { "name": "laravel/pint", - "version": "v1.27.1", + "version": "v1.29.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "54cca2de13790570c7b6f0f94f37896bee4abcb5" + "reference": "0770e9b7fafd50d4586881d456d6eb41c9247a80" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/54cca2de13790570c7b6f0f94f37896bee4abcb5", - "reference": "54cca2de13790570c7b6f0f94f37896bee4abcb5", + "url": "https://api.github.com/repos/laravel/pint/zipball/0770e9b7fafd50d4586881d456d6eb41c9247a80", + "reference": "0770e9b7fafd50d4586881d456d6eb41c9247a80", "shasum": "" }, "require": { @@ -2116,13 +2116,14 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.93.1", - "illuminate/view": "^12.51.0", - "larastan/larastan": "^3.9.2", - "laravel-zero/framework": "^12.0.5", + "friendsofphp/php-cs-fixer": "^3.95.1", + "illuminate/view": "^12.56.0", + "larastan/larastan": "^3.9.6", + "laravel-zero/framework": "^12.1.0", "mockery/mockery": "^1.6.12", - "nunomaduro/termwind": "^2.3.3", - "pestphp/pest": "^3.8.5" + "nunomaduro/termwind": "^2.4.0", + "pestphp/pest": "^3.8.6", + "shipfastlabs/agent-detector": "^1.1.3" }, "bin": [ "builds/pint" @@ -2159,7 +2160,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2026-02-10T20:00:20+00:00" + "time": "2026-04-20T15:26:14+00:00" }, { "name": "myclabs/deep-copy", @@ -3902,5 +3903,5 @@ "ext-simplexml": "*" }, "platform-dev": {}, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } From 9ec198319e59ff0a3cfca6341ce5fadc4b29881f Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:48:14 +0400 Subject: [PATCH 5/9] fix: use php 8.3 with dynamic extension paths in Dockerfile --- Dockerfile | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0bf6ba47..f19e93b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN composer install \ --no-scripts \ --prefer-dist -FROM php:8.1-cli-alpine AS compile +FROM php:8.3-cli-alpine AS compile ENV PHP_ZSTD_VERSION="master" ENV PHP_BROTLI_VERSION="7ae4fcd8b81a65d7521c298cae49af386d1ea4e3" @@ -39,7 +39,9 @@ RUN git clone --recursive --depth 1 --branch $PHP_ZSTD_VERSION https://github.co && cd php-ext-zstd \ && phpize \ && ./configure --with-libzstd \ - && make && make install + && make && make install \ + && mkdir -p /ext \ + && cp $(php-config --extension-dir)/zstd.so /ext/zstd.so ## Brotli Extension FROM compile AS brotli @@ -48,7 +50,9 @@ RUN git clone https://github.com/kjdev/php-ext-brotli.git \ && git reset --hard $PHP_BROTLI_VERSION \ && phpize \ && ./configure --with-libbrotli \ - && make && make install + && make && make install \ + && mkdir -p /ext \ + && cp $(php-config --extension-dir)/brotli.so /ext/brotli.so ## LZ4 Extension FROM compile AS lz4 @@ -57,7 +61,9 @@ RUN git clone --recursive https://github.com/kjdev/php-ext-lz4.git \ && git reset --hard $PHP_LZ4_VERSION \ && phpize \ && ./configure --with-lz4-includedir=/usr \ - && make && make install + && make && make install \ + && mkdir -p /ext \ + && cp $(php-config --extension-dir)/lz4.so /ext/lz4.so ## Snappy Extension FROM compile AS snappy @@ -66,7 +72,9 @@ RUN git clone --recursive https://github.com/kjdev/php-ext-snappy.git \ && git reset --hard $PHP_SNAPPY_VERSION \ && phpize \ && ./configure \ - && make && make install + && make && make install \ + && mkdir -p /ext \ + && cp $(php-config --extension-dir)/snappy.so /ext/snappy.so ## Xz Extension FROM compile AS xz @@ -85,7 +93,9 @@ RUN git clone https://github.com/codemasher/php-ext-xz.git --branch ${PHP_EXT_XZ && cd php-ext-xz \ && phpize \ && ./configure \ - && make && make install + && make && make install \ + && mkdir -p /ext \ + && cp $(php-config --extension-dir)/xz.so /ext/xz.so FROM compile AS final @@ -93,22 +103,25 @@ LABEL maintainer="team@appwrite.io" WORKDIR /usr/src/code -RUN echo extension=zstd.so >> /usr/local/etc/php/conf.d/zstd.ini -RUN echo extension=brotli.so >> /usr/local/etc/php/conf.d/brotli.ini -RUN echo extension=lz4.so >> /usr/local/etc/php/conf.d/lz4.ini -RUN echo extension=snappy.so >> /usr/local/etc/php/conf.d/snappy.ini -RUN echo extension=xz.so >> /usr/local/etc/php/conf.d/xz.ini - RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" \ && echo "opcache.enable_cli=1" >> $PHP_INI_DIR/php.ini \ && echo "memory_limit=1024M" >> $PHP_INI_DIR/php.ini COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor -COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20210902/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20210902/ -COPY --from=brotli /usr/local/lib/php/extensions/no-debug-non-zts-20210902/brotli.so /usr/local/lib/php/extensions/no-debug-non-zts-20210902/ -COPY --from=lz4 /usr/local/lib/php/extensions/no-debug-non-zts-20210902/lz4.so /usr/local/lib/php/extensions/no-debug-non-zts-20210902/ -COPY --from=snappy /usr/local/lib/php/extensions/no-debug-non-zts-20210902/snappy.so /usr/local/lib/php/extensions/no-debug-non-zts-20210902/ -COPY --from=xz /usr/local/lib/php/extensions/no-debug-non-zts-20210902/xz.so /usr/local/lib/php/extensions/no-debug-non-zts-20210902/ +COPY --from=zstd /ext/zstd.so /ext/ +COPY --from=brotli /ext/brotli.so /ext/ +COPY --from=lz4 /ext/lz4.so /ext/ +COPY --from=snappy /ext/snappy.so /ext/ +COPY --from=xz /ext/xz.so /ext/ + +RUN EXT_DIR=$(php-config --extension-dir) \ + && mkdir -p $EXT_DIR \ + && mv /ext/*.so $EXT_DIR/ \ + && echo extension=zstd.so >> /usr/local/etc/php/conf.d/zstd.ini \ + && echo extension=brotli.so >> /usr/local/etc/php/conf.d/brotli.ini \ + && echo extension=lz4.so >> /usr/local/etc/php/conf.d/lz4.ini \ + && echo extension=snappy.so >> /usr/local/etc/php/conf.d/snappy.ini \ + && echo extension=xz.so >> /usr/local/etc/php/conf.d/xz.ini # Add Source Code COPY . /usr/src/code From 004018fbdfca8418b5734be9269fe76656abbf7c Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:49:16 +0400 Subject: [PATCH 6/9] chore: run formatter --- src/Storage/Device/Local.php | 4 ++-- src/Storage/Device/S3.php | 22 +++++++++++----------- tests/Storage/Device/WasabiTest.php | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 59f44abf..43c87ee2 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -52,7 +52,7 @@ public function getPath(string $filename, ?string $prefix = null): string * return number of chunks uploaded or 0 if it fails. * * - * @throws \Exception + * @throws Exception */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { @@ -104,7 +104,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks * @param int chunk * @param int chunks * - * @throws \Exception + * @throws Exception */ public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 50558f60..c837d83f 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -163,7 +163,7 @@ public static function setRetryDelay(int $delay): void * @param int chunk * @param int chunks * - * @throws \Exception + * @throws Exception */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { @@ -180,7 +180,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks * @param int chunk * @param int chunks * - * @throws \Exception + * @throws Exception */ public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { @@ -248,7 +248,7 @@ public function transfer(string $path, string $destination, Device $device): boo * Initiate a multipart upload and return an upload ID. * * - * @throws \Exception + * @throws Exception */ protected function createMultipartUpload(string $path, string $contentType): string { @@ -268,7 +268,7 @@ protected function createMultipartUpload(string $path, string $contentType): str * * @param string $source * - * @throws \Exception + * @throws Exception */ protected function uploadPart(string $data, string $path, string $contentType, int $chunk, string $uploadId): string { @@ -291,7 +291,7 @@ protected function uploadPart(string $data, string $path, string $contentType, i * Complete Multipart Upload * * - * @throws \Exception + * @throws Exception */ protected function completeMultipartUpload(string $path, string $uploadId, array $parts): bool { @@ -314,7 +314,7 @@ protected function completeMultipartUpload(string $path, string $uploadId, array * Abort Chunked Upload * * - * @throws \Exception + * @throws Exception */ public function abort(string $path, string $extra = ''): bool { @@ -332,7 +332,7 @@ public function abort(string $path, string $extra = ''): bool * @param int offset * @param int|null length * - * @throws \Exception + * @throws Exception */ public function read(string $path, int $offset = 0, ?int $length = null): string { @@ -354,7 +354,7 @@ public function read(string $path, int $offset = 0, ?int $length = null): string * Write file by given path. * * - * @throws \Exception + * @throws Exception */ public function write(string $path, string $data, string $contentType = ''): bool { @@ -375,7 +375,7 @@ public function write(string $path, string $data, string $contentType = ''): boo * * @see http://php.net/manual/en/function.filesize.php * - * @throws \Exception + * @throws Exception */ public function delete(string $path, bool $recursive = false): bool { @@ -429,7 +429,7 @@ protected function listObjects(string $prefix = '', int $maxKeys = self::MAX_PAG * Delete files in given path, path must be a directory. Return true on success and false on failure. * * - * @throws \Exception + * @throws Exception */ public function deletePath(string $path): bool { @@ -677,7 +677,7 @@ private function getSignatureV4(string $method, string $uri, array $parameters = * * @return object * - * @throws \Exception + * @throws Exception */ protected function call(string $operation, string $method, string $uri, string $data = '', array $parameters = [], bool $decode = true) { diff --git a/tests/Storage/Device/WasabiTest.php b/tests/Storage/Device/WasabiTest.php index fabb2759..6acd3d80 100644 --- a/tests/Storage/Device/WasabiTest.php +++ b/tests/Storage/Device/WasabiTest.php @@ -14,7 +14,7 @@ protected function init(): void $secret = $_SERVER['WASABI_SECRET'] ?? ''; $bucket = 'utopia-storage-tests'; - $this->object = new Wasabi($this->root, $key, $secret, $bucket, Wasabi::EU_CENTRAL_1, WASABI::ACL_PRIVATE); + $this->object = new Wasabi($this->root, $key, $secret, $bucket, Wasabi::EU_CENTRAL_1, Wasabi::ACL_PRIVATE); } protected function getAdapterName(): string From dd7bf9746a986a7d215579e4e457ab193c7fe958 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 14:59:29 +0400 Subject: [PATCH 7/9] fix: sort parts by part number before completing S3 multipart upload --- src/Storage/Device/S3.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index c837d83f..d2e79c4e 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -297,6 +297,8 @@ protected function completeMultipartUpload(string $path, string $uploadId, array { $uri = $path !== '' ? '/'.\str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + \ksort($parts); + $body = ''; foreach ($parts as $key => $etag) { $body .= "{$etag}{$key}"; From a42a2c616251c0f00ab6d9daabcd03cb21b6d579 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 15:05:58 +0400 Subject: [PATCH 8/9] fix: escape glob metacharacters in countChunks filename pattern --- src/Storage/Device/Local.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 43c87ee2..f8cb17d4 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -143,7 +143,8 @@ public function uploadData(string $data, string $path, string $contentType, int private function countChunks(string $tmp, string $path): int { - $pattern = $tmp.DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.*'; + $filename = \str_replace(['\\', '*', '?', '[', ']', '{', '}'], ['\\\\', '\\*', '\\?', '\\[', '\\]', '\\{', '\\}'], \pathinfo($path, PATHINFO_FILENAME)); + $pattern = $tmp.DIRECTORY_SEPARATOR.$filename.'.part.*'; $files = \glob($pattern); if ($files === false) { return 0; From d1d3de0908dc0f6c9730a3c8bf1270b89acaa41f Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 27 Apr 2026 15:11:24 +0400 Subject: [PATCH 9/9] fix: escape glob metacharacters in both tmp directory and filename for countChunks --- src/Storage/Device/Local.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index f8cb17d4..c25612d9 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -143,8 +143,10 @@ public function uploadData(string $data, string $path, string $contentType, int private function countChunks(string $tmp, string $path): int { - $filename = \str_replace(['\\', '*', '?', '[', ']', '{', '}'], ['\\\\', '\\*', '\\?', '\\[', '\\]', '\\{', '\\}'], \pathinfo($path, PATHINFO_FILENAME)); - $pattern = $tmp.DIRECTORY_SEPARATOR.$filename.'.part.*'; + $escaped = function (string $literal): string { + return \str_replace(['\\', '*', '?', '[', ']', '{', '}'], ['\\\\', '\\*', '\\?', '\\[', '\\]', '\\{', '\\}'], $literal); + }; + $pattern = $escaped($tmp).DIRECTORY_SEPARATOR.$escaped(\pathinfo($path, PATHINFO_FILENAME)).'.part.*'; $files = \glob($pattern); if ($files === false) { return 0;