feat: add concurrent uploads for uploads:import command

ymirapp · Jul 8, 2022 · c08a4d4 · c08a4d4
1 parent d01dd44
commit c08a4d4
Showing 1 changed file with 18 additions and 71 deletions.
diff --git a/src/Command/Uploads/ImportUploadsCommand.php b/src/Command/Uploads/ImportUploadsCommand.php
@@ -45,13 +45,6 @@ class ImportUploadsCommand extends AbstractProjectCommand
      */
     public const NAME = 'uploads:import';
 
-    /**
-     * The temporary directory used for importing.
-     *
-     * @var string
-     */
-    private $tempDirectory;
-
     /**
      * The uploader used to upload files.
      *
@@ -69,25 +62,6 @@ public function __construct(ApiClient $apiClient, CliConfiguration $cliConfigura
         $this->uploader = $uploader;
     }
 
-    /**
-     * Delete the temporary directory when we're done the execution.
-     */
-    public function __destruct()
-    {
-        if (!is_string($this->tempDirectory) || !is_dir($this->tempDirectory)) {
-            return;
-        }
-
-        $files = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($this->tempDirectory, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST);
-
-        foreach ($files as $file) {
-            $function = $file->isDir() ? 'rmdir' : 'unlink';
-            $function($file->getRealPath());
-        }
-
-        rmdir($this->tempDirectory);
-    }
-
     /**
      * {@inheritdoc}
      */
@@ -99,17 +73,24 @@ protected function configure()
             ->addArgument('path', InputArgument::REQUIRED, 'The path to the files to import')
             ->addOption('environment', null, InputOption::VALUE_REQUIRED, 'The environment to upload files to', 'staging')
             ->addOption('force', null, InputOption::VALUE_NONE, 'Force the import to run')
-            ->addOption('size', null, InputOption::VALUE_REQUIRED, 'The number of files to process at a time', '20');
+            ->addOption('size', null, InputOption::VALUE_REQUIRED, 'The number of files to process at a time');
     }
 
     /**
      * {@inheritdoc}
      */
     protected function perform(InputInterface $input, OutputInterface $output)
     {
+        $adapter = $this->getAdapter($this->getStringArgument($input, 'path'));
         $environment = (string) $this->getStringOption($input, 'environment');
-        $filesystem = new Filesystem($this->getAdapter($this->getStringArgument($input, 'path')));
-        $size = (int) $this->getNumericOption($input, 'size');
+        $filesystem = new Filesystem($adapter);
+        $size = $this->getNumericOption($input, 'size');
+
+        if (null === $size && $adapter instanceof LocalFilesystemAdapter) {
+            $size = 1000;
+        } elseif (null === $size) {
+            $size = 100;
+        }
 
         if ($size < 1) {
             throw new InvalidArgumentException('Cannot have a "size" smaller than 1');
@@ -119,8 +100,6 @@ protected function perform(InputInterface $input, OutputInterface $output)
             return;
         }
 
-        $this->tempDirectory = $this->createTempDirectory();
-
         $output->info(sprintf('Starting file import to the "<comment>%s</comment>" environment "uploads" directory', $environment));
 
         $progressBar = new ProgressBar($output);
@@ -129,7 +108,7 @@ protected function perform(InputInterface $input, OutputInterface $output)
         $total = 0;
         $progressBar->setMessage((string) $total, 'total');
 
-        LazyCollection::make(function () use ($filesystem) {
+        $requests = LazyCollection::make(function () use ($filesystem) {
             $files = $filesystem->listContents('', Filesystem::LIST_DEEP)->filter(function (StorageAttributes $attributes) {
                 return $attributes->isFile();
             });
@@ -139,53 +118,21 @@ protected function perform(InputInterface $input, OutputInterface $output)
             }
         })->chunk($size)->mapWithKeys(function (Enumerable $chunkedFiles) use ($environment) {
             return $this->getSignedUploadRequest($environment, $chunkedFiles);
-        })->each(function (array $request, string $filePath) use ($filesystem, $progressBar, &$total) {
-            $tempFilePath = $this->tempDirectory.'/'.basename($filePath);
-
-            $progressBar->setMessage($filePath, 'filename');
-            $progressBar->advance();
-
-            file_put_contents($tempFilePath, $filesystem->readStream($filePath));
-
-            $this->uploader->uploadFile($tempFilePath, $request['uri'], $request['headers']);
-
-            unlink($tempFilePath);
+        })->map(function (array $request, string $filePath) use ($filesystem, $progressBar, &$total) {
+            $request['body'] = $filesystem->readStream($filePath);
 
             ++$total;
 
+            $progressBar->setMessage($filePath, 'filename');
             $progressBar->setMessage((string) $total, 'total');
             $progressBar->advance();
-        });
-
-        $output->info(sprintf('Files imported successfully to the "<comment>%s</comment>" environment "uploads" directory', $environment));
-    }
 
-    /**
-     * Create a temporary directory to copy over files temporarily.
-     */
-    private function createTempDirectory(): string
-    {
-        $baseDirectory = sys_get_temp_dir().'/';
-        $maxAttempts = 100;
-
-        if (!is_dir($baseDirectory)) {
-            throw new RuntimeException(sprintf('"%s" isn\'t a directory', $baseDirectory));
-        } elseif (!is_writable($baseDirectory)) {
-            throw new RuntimeException(sprintf('"%s" isn\'t writable', $baseDirectory));
-        }
-
-        $attempts = 0;
+            return $request;
+        });
 
-        do {
-            ++$attempts;
-            $tmpDirectory = sprintf('%s%s%s', $baseDirectory, 'ymir_', mt_rand(100000, mt_getrandmax()));
-        } while (!mkdir($tmpDirectory) && $attempts < $maxAttempts);
+        $this->uploader->batch('PUT', $requests);
 
-        if (!is_dir($tmpDirectory)) {
-            throw new RuntimeException('Failed to create a temporary directory');
-        }
-
-        return $tmpDirectory;
+        $output->info(sprintf('Files imported successfully to the "<comment>%s</comment>" environment "uploads" directory', $environment));
     }
 
     /**