[Translation][Lokalise] fix "Project too big for sync export"

Author: santysisi

src/Symfony/Component/Translation/Bridge/Lokalise/LokaliseProvider.php

@@ -12,6 +12,7 @@
 namespace Symfony\Component\Translation\Bridge\Lokalise;
 
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Translation\Exception\LogicException;
 use Symfony\Component\Translation\Exception\ProviderException;
 use Symfony\Component\Translation\Loader\LoaderInterface;
 use Symfony\Component\Translation\MessageCatalogueInterface;
@@ -31,6 +32,7 @@
 final class LokaliseProvider implements ProviderInterface
 {
     private const LOKALISE_GET_KEYS_LIMIT = 5000;
+    private const PROJECT_TOO_BIG_STATUS_CODE = 413;
 
     private HttpClientInterface $client;
     private LoaderInterface $loader;
@@ -165,6 +167,13 @@ private function exportFiles(array $locales, array $domains): array
         }
 
         if (200 !== $response->getStatusCode()) {
+            if (($errorCode = $responseContent['error']['code'] ?? null) && self::PROJECT_TOO_BIG_STATUS_CODE === $errorCode) {
+                if (!\extension_loaded('zip')) {
+                    throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s". There are too many translations. You need to enable the "zip" extension to use asynchronous export.', $response->getContent(false)), $response);
+                }
+
+                return $this->exportFilesAsync($locales, $domains);
+            }
             throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
         }
 
@@ -176,6 +185,109 @@ private function exportFiles(array $locales, array $domains): array
         return array_combine($reformattedLanguages, $responseContent['files']);
     }
 
+    /**
+     * @see https://developers.lokalise.com/reference/download-files-async
+     */
+    private function exportFilesAsync(array $locales, array $domains): array
+    {
+        $response = $this->client->request('POST', 'files/async-download', [
+            'json' => [
+                'format' => 'symfony_xliff',
+                'original_filenames' => true,
+                'filter_langs' => array_values($locales),
+                'filter_filenames' => array_map($this->getLokaliseFilenameFromDomain(...), $domains),
+                'export_empty_as' => 'skip',
+                'replace_breaks' => false,
+            ],
+        ]);
+
+        if (200 !== $response->getStatusCode()) {
+            throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
+        }
+
+        $processId = $response->toArray()['process_id'];
+        $attempt = 0;
+        while (true) {
+            $response = $this->client->request('GET', \sprintf('processes/%s', $processId));
+            $process = $response->toArray()['process'];
+            if ('failed' === $process['status']) {
+                throw new ProviderException(\sprintf('Unable to export translations from Lokalise: "%s".', $response->getContent(false)), $response);
+            }
+            if ('finished' === $process['status']) {
+                $downloadUrl = $process['details']['download_url'];
+                break;
+            }
+            ++$attempt;
+            usleep(500000 * $attempt);
+        }
+
+        $newfile = \sprintf('%s%s%s.zip', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
+        $extractPath = \sprintf('%s%s%s', sys_get_temp_dir(), \DIRECTORY_SEPARATOR, uniqid());
+        if (!copy($downloadUrl, $newfile)) {
+            throw new LogicException(\sprintf('failed to copy "%s".', $downloadUrl));
+        }
+
+        try {
+            $zip = new \ZipArchive();
+            if (!$zip->open($newfile, \ZipArchive::CREATE)) {
+                throw new LogicException(\sprintf('failed to open zip file "%s".', $newfile));
+            }
+
+            try {
+                if (!$zip->extractTo($extractPath)) {
+                    throw new LogicException(\sprintf('failed to extract content from zip file "%s".', $newfile));
+                }
+            } finally {
+                $zip->close();
+            }
+
+            $fileContents = $this->getFileContents($extractPath);
+
+            // Lokalise returns languages with "-" separator, we need to reformat them to "_" separator.
+            /** @var array<array-key, array-key> $reformattedLanguages */
+            $reformattedLanguages = array_map(function ($language) {
+                return str_replace('-', '_', $language);
+            }, array_keys($fileContents));
+
+            return array_combine($reformattedLanguages, $fileContents);
+        } finally {
+            unlink($newfile);
+            $this->removeDir($extractPath);
+        }
+    }
+
+    private function getFileContents(string $dir, string $baseDir = ''): array
+    {
+        $fileContents = [];
+        foreach (scandir($dir) as $filename) {
+            if (\in_array($filename, ['.', '..'])) {
+                continue;
+            }
+            $path = \sprintf('%s%s%s', $dir, \DIRECTORY_SEPARATOR, $filename);
+            if (is_dir($path)) {
+                $fileContents = array_merge($fileContents, $this->getFileContents($path, $filename));
+                continue;
+            }
+            $fileContents[$baseDir][$filename]['content'] = file_get_contents($path);
+        }
+
+        return $fileContents;
+    }
+
+    private function removeDir(string $dir): void
+    {
+        $it = new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS);
+        $files = new \RecursiveIteratorIterator($it, \RecursiveIteratorIterator::CHILD_FIRST);
+        foreach ($files as $file) {
+            if ($file->isDir()) {
+                rmdir($file->getPathname());
+            } else {
+                unlink($file->getPathname());
+            }
+        }
+        rmdir($dir);
+    }
+
     private function createKeys(array $keys, string $domain): array
     {
         $keysToCreate = [];

src/Symfony/Component/Translation/Bridge/Lokalise/Tests/Fixtures/Symfony-locale.zip

@@ -697,6 +697,94 @@ public function testReadForManyLocalesAndManyDomains(array $locales, array $doma
         }
     }
 
+    /**
+     * @requires extension zip
+     */
+    public function testReadWithExportAsync()
+    {
+        $firstResponse = function (): ResponseInterface {
+            return new JsonMockResponse(
+                ['error' => ['code' => 413, 'message' => 'test']],
+                ['http_code' => 406],
+            );
+        };
+        $secondResponse = function (): ResponseInterface {
+            return new JsonMockResponse(
+                ['process_id' => 123],
+            );
+        };
+        $thirdResponse = function (): ResponseInterface {
+            $zipLocation = __DIR__.\DIRECTORY_SEPARATOR.'Fixtures'.\DIRECTORY_SEPARATOR.'Symfony-locale.zip';
+
+            return new JsonMockResponse(
+                ['process' => ['status' => 'finished', 'details' => ['download_url' => $zipLocation]]],
+            );
+        };
+
+        $provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
+            'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
+            'headers' => ['X-Api-Token' => 'API_KEY'],
+        ]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
+        $translatorBag = $provider->read(['foo'], ['baz']);
+
+        // We don't want to assert equality of metadata here, due to the ArrayLoader usage.
+        /** @var MessageCatalogue $catalogue */
+        foreach ($translatorBag->getCatalogues() as $catalogue) {
+            $catalogue->deleteMetadata('', '');
+        }
+
+        $arrayLoader = new ArrayLoader();
+        $expectedTranslatorBag = new TranslatorBag();
+        $expectedTranslatorBag->addCatalogue($arrayLoader->load(
+            [
+                'how are you' => 'How are you?',
+                'welcome_header' => 'Hello world!',
+            ],
+            'en',
+            'no_filename'
+        ));
+        $expectedTranslatorBag->addCatalogue($arrayLoader->load(
+            [
+                'how are you' => 'Como estas?',
+                'welcome_header' => 'Hola mundo!',
+            ],
+            'es',
+            'no_filename'
+        ));
+        $this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
+    }
+
+    /**
+     * @requires extension zip
+     */
+    public function testReadWithExportAsyncFailedProcess()
+    {
+        $firstResponse = function (): ResponseInterface {
+            return new JsonMockResponse(
+                ['error' => ['code' => 413, 'message' => 'test']],
+                ['http_code' => 406],
+            );
+        };
+        $secondResponse = function (): ResponseInterface {
+            return new JsonMockResponse(
+                ['process_id' => 123],
+            );
+        };
+        $thirdResponse = function (): ResponseInterface {
+            return new JsonMockResponse(
+                ['process' => ['status' => 'failed']],
+            );
+        };
+
+        $provider = self::createProvider((new MockHttpClient([$firstResponse, $secondResponse, $thirdResponse]))->withOptions([
+            'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
+            'headers' => ['X-Api-Token' => 'API_KEY'],
+        ]), new XliffFileLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
+
+        $this->expectException(ProviderException::class);
+        $provider->read(['foo'], ['baz']);
+    }
+
     public function testDeleteProcess()
     {
         $getKeysIdsForMessagesDomainResponse = function (string $method, string $url, array $options = []): ResponseInterface {