Initial commit

Classes/Command/MediaCommandController.php

@@ -0,0 +1,81 @@
+<?php
+declare(strict_types=1);
+
+namespace Wwwision\AssetSync\Command;
+
+use Neos\Flow\Annotations as Flow;
+use Neos\Flow\Cli\CommandController;
+use Neos\Utility\Arrays;
+use Wwwision\AssetSync\SyncService;
+use Wwwision\AssetSync\ValueObject\Preset;
+use Wwwision\BatchProcessing\BatchProcessRunner;
+use Wwwision\BatchProcessing\ProgressHandler\NullProgressHandler;
+use Wwwision\BatchProcessing\ProgressHandler\ProgressBarRenderer;
+use Wwwision\BatchProcessing\ProgressPipe;
+
+final class MediaCommandController extends CommandController
+{
+
+    /**
+     * @Flow\Inject(lazy=false)
+     * @var SyncService
+     */
+    protected SyncService $syncService;
+
+    /**
+     * Update imported asset metadata and (optionally) resources from their original asset source
+     *
+     * @param string|null $assetSources Comma separated list of Asset Source identifiers to sync. If null, all imported assets are synced.
+     * @param bool $synchronizeResources If set, resource binaries are synchronized (this can be slow!). Otherwise, only the asset metadata is updated.
+     * @param int|null $batchSize Number of assets to synchronize in a single run. Larger numbers can increase performance but may lead to high memory consumption
+     * @param int|null $poolSize Maximum number of sub processes to run at the same time
+     * @param bool $quiet If set only the number of errors is outputted (if any)
+     */
+    public function synchronizeImportedAssetsCommand(string $assetSources = null, bool $synchronizeResources = false, int $batchSize = null, int $poolSize = null, bool $quiet = false): void
+    {
+        $preset = new Preset($synchronizeResources);
+        if ($assetSources !== null) {
+            $preset = $preset->filterAssetSourceIdentifiers(Arrays::trimExplode(',', $assetSources));
+        }
+        $numberOfAssetsToSync = $this->syncService->numberOfAssets($preset);
+
+        $quiet || $this->outputLine('Synchronizing metadata%s of <b>%d</b> assets...', [$preset->synchronizeResources ? ' and resources' : '', $numberOfAssetsToSync]);
+        $progressHandler = $quiet ? new NullProgressHandler() : ProgressBarRenderer::create($this->output->getOutput());
+        $runner = new BatchProcessRunner('wwwision.assetsync:media:synchronizeimportedassetsbatch', ['presetJson' => $preset->toJson(), 'offset' => '{offset}', 'limit' => '{limit}'], $progressHandler);
+        if ($batchSize !== null) {
+            $runner->setBatchSize($batchSize);
+        }
+        if ($poolSize !== null) {
+            $runner->setPoolSize($poolSize);
+        }
+        $runner->onFinish(function(array $errors) use ($quiet) {
+            if ($errors === []) {
+                $quiet || $this->outputLine('<success>Done</success>');
+                return;
+            }
+            $this->outputLine('<error>Finished with <b>%d</b> error%s%s</error>', [\count($errors), \count($errors) === 1 ? '' : 's', $quiet ? '' : ':']);
+            if (!$quiet) {
+                foreach ($errors as $error) {
+                    $this->outputLine('  %s', [$error]);
+                }
+            }
+            exit(1);
+        });
+        $runner->start($numberOfAssetsToSync);
+    }
+
+    /**
+     * @param string $presetJson JSON encoded preset
+     * @param int $offset zero based offset
+     * @param int $limit size of the batch to import
+     * @internal
+     */
+    public function synchronizeImportedAssetsBatchCommand(string $presetJson, int $offset, int $limit): void
+    {
+        $preset = Preset::fromJson($presetJson);
+        $processPipe = new ProgressPipe();
+        $this->syncService->onError(fn($message) => $processPipe->error($message));
+        $this->syncService->onProgress(fn() => $processPipe->advance());
+        $this->syncService->syncBatch($preset, $offset, $limit);
+    }
+}

Classes/SyncService.php

@@ -0,0 +1,176 @@
+<?php
+declare(strict_types=1);
+
+namespace Wwwision\AssetSync;
+
+use Neos\Flow\Persistence\QueryInterface;
+use Neos\Flow\ResourceManagement\ResourceManager;
+use Neos\Media\Domain\Model\AssetInterface;
+use Neos\Media\Domain\Model\AssetSource\AssetProxy\AssetProxyInterface;
+use Neos\Media\Domain\Model\AssetSource\AssetProxy\SupportsIptcMetadataInterface;
+use Neos\Media\Domain\Model\ImageVariant;
+use Neos\Media\Domain\Model\ImportedAsset;
+use Neos\Media\Domain\Repository\AssetRepository;
+use Neos\Media\Domain\Repository\ImportedAssetRepository;
+use Neos\Media\Domain\Service\AssetService;
+use Neos\Media\Domain\Service\AssetSourceService;
+use Wwwision\AssetSync\ValueObject\Preset;
+
+final class SyncService
+{
+    private ImportedAssetRepository $importedAssetRepository;
+    private AssetSourceService $assetSourceService;
+    private AssetRepository $assetRepository;
+    private AssetService $assetService;
+    private ResourceManager $resourceManager;
+
+    private const EVENT_START = 'start';
+    private const EVENT_PROGRESS = 'progress';
+    private const EVENT_ERROR = 'error';
+
+    private array $eventHandlers = [];
+
+    public function __construct(ImportedAssetRepository $importedAssetRepository, AssetSourceService $assetSourceService, AssetRepository $assetRepository, AssetService $assetService, ResourceManager $resourceManager)
+    {
+        $this->importedAssetRepository = $importedAssetRepository;
+        $this->assetSourceService = $assetSourceService;
+        $this->assetRepository = $assetRepository;
+        $this->assetService = $assetService;
+        $this->resourceManager = $resourceManager;
+    }
+
+    public function numberOfAssets(Preset $preset): int
+    {
+        return $this->importedAssetsForPreset($preset)->count();
+    }
+
+
+    public function syncBatch(Preset $preset, int $offset, ?int $limit): void
+    {
+        $importedAssetsBatch = $this->importedAssetsForPreset($preset)->setOffset($offset)->setLimit($limit)->execute();
+        $this->dispatch(self::EVENT_START, $importedAssetsBatch);
+        /** @var ImportedAsset $importedAsset */
+        foreach ($importedAssetsBatch as $importedAsset) {
+            $this->updateAsset($preset, $importedAsset);
+        }
+    }
+
+    private function updateAsset(Preset $preset, ImportedAsset $importedAsset): void
+    {
+        /** @var AssetInterface $localAsset */
+        $localAsset = $this->assetRepository->findByIdentifier($importedAsset->getLocalAssetIdentifier());
+        if ($localAsset === null) {
+            $this->dispatch(self::EVENT_ERROR, sprintf('Failed to load local asset with id "%s"', $importedAsset->getLocalAssetIdentifier()));
+            return;
+        }
+
+        $assetSource = $this->assetSourceService->getAssetSources()[$importedAsset->getAssetSourceIdentifier()] ?? null;
+        if ($assetSource === null) {
+            $this->dispatch(self::EVENT_ERROR, sprintf('Asset source id "%s" referenced in imported asset "%s" does not exist', $importedAsset->getAssetSourceIdentifier(), $importedAsset->getLocalAssetIdentifier()));
+            return;
+        }
+        try {
+            $assetProxy = $assetSource->getAssetProxyRepository()->getAssetProxy($importedAsset->getRemoteAssetIdentifier());
+        } catch (\Throwable $e) {
+            $this->dispatch(self::EVENT_ERROR, sprintf('Failed to retrieve asset proxy for imported asset "%s": %s', $importedAsset->getLocalAssetIdentifier(), $e->getMessage()));
+            return;
+        }
+        try {
+            $metadataWasUpdated = $this->updateAssetPropertiesFromIptcMetadata($localAsset, $assetProxy);
+        } catch (\Throwable $e) {
+            $this->dispatch(self::EVENT_ERROR, sprintf('Failed to update asset properties for imported asset "%s": %s', $importedAsset->getLocalAssetIdentifier(), $e->getMessage()));
+        }
+        try {
+            $resourceWasReplaced = $preset->synchronizeResources && $this->replaceAssetResource($localAsset, $assetProxy);
+        } catch (\Throwable $e) {
+            $this->dispatch(self::EVENT_ERROR, sprintf('Failed to replace resource for imported asset "%s": %s', $importedAsset->getLocalAssetIdentifier(), $e->getMessage()));
+        }
+        $this->dispatch(self::EVENT_PROGRESS, $importedAsset, $metadataWasUpdated, $resourceWasReplaced);
+    }
+
+    public function onStart(callable $handler): void
+    {
+        $this->on(self::EVENT_START, $handler);
+    }
+
+    public function onProgress(callable $handler): void
+    {
+        $this->on(self::EVENT_PROGRESS, $handler);
+    }
+
+    public function onError(callable $handler): void
+    {
+        $this->on(self::EVENT_ERROR, $handler);
+    }
+
+    /* ----------------------------- */
+
+
+    private function updateAssetPropertiesFromIptcMetadata(AssetInterface $localAsset, AssetProxyInterface $assetProxy): bool
+    {
+        if (!$assetProxy instanceof SupportsIptcMetadataInterface) {
+            return false;
+        }
+        $assetIsUpdated = false;
+        if (!$localAsset instanceof ImageVariant && $assetProxy->getIptcProperty('Title') !== $localAsset->getTitle()) {
+            $localAsset->setTitle($assetProxy->getIptcProperty('Title'));
+            $assetIsUpdated = true;
+        }
+        if ($assetProxy->getIptcProperty('CaptionAbstract') !== $localAsset->getCaption()) {
+            $localAsset->setCaption($assetProxy->getIptcProperty('CaptionAbstract'));
+            $assetIsUpdated = true;
+        }
+        if ($assetProxy->getIptcProperty('CopyrightNotice') !== $localAsset->getCopyrightNotice()) {
+            $localAsset->setCopyrightNotice($assetProxy->getIptcProperty('CopyrightNotice'));
+            $assetIsUpdated = true;
+        }
+        if ($assetIsUpdated) {
+            $this->assetRepository->update($localAsset);
+        }
+        return $assetIsUpdated;
+    }
+
+    private function replaceAssetResource(AssetInterface $localAsset, AssetProxyInterface $assetProxy): bool
+    {
+        if ($localAsset instanceof ImageVariant) {
+            return false;
+        }
+        $resourceWasReplaced = false;
+        $assetResource = $this->resourceManager->importResource($assetProxy->getImportStream());
+        if ($assetResource->getFilename() !== $assetProxy->getFilename()) {
+            $assetResource->setFilename($assetProxy->getFilename());
+            $resourceWasReplaced = true;
+        }
+        $this->assetService->replaceAssetResource($localAsset, $assetResource);
+        return $resourceWasReplaced;
+    }
+
+    private function importedAssetsForPreset(Preset $preset): QueryInterface
+    {
+        $query = $this->importedAssetRepository->createQuery();
+        if ($preset->hasAssetSourceFilter()) {
+            $query = $query->matching($query->in('assetSourceIdentifier', $preset->assetSourceIdentifiers));
+        }
+        return $query;
+    }
+
+    private function on(string $event, callable $handler): void
+    {
+        if (!isset($this->eventHandlers[$event])) {
+            $this->eventHandlers[$event] = [];
+        }
+        $this->eventHandlers[$event][] = $handler;
+    }
+
+
+    private function dispatch(string $event, ...$arguments): void
+    {
+        if (!isset($this->eventHandlers[$event])) {
+            return;
+        }
+        foreach ($this->eventHandlers[$event] as $handler) {
+            $handler(...$arguments);
+        }
+    }
+
+}

Classes/ValueObject/Preset.php

@@ -0,0 +1,66 @@
+<?php
+declare(strict_types=1);
+
+namespace Wwwision\AssetSync\ValueObject;
+
+use Neos\Flow\Annotations as Flow;
+
+/**
+ * @Flow\Proxy(false)
+ */
+final class Preset implements \JsonSerializable
+{
+    public array $assetSourceIdentifiers = [];
+    public bool $synchronizeResources;
+
+    public function __construct(bool $synchronizeResources)
+    {
+        $this->synchronizeResources = $synchronizeResources;
+    }
+
+    public static function fromJson(string $json): self
+    {
+        try {
+            $data = json_decode($json, true, 512, JSON_THROW_ON_ERROR);
+        } catch (\JsonException $e) {
+            throw new \InvalidArgumentException(sprintf('Failed to decode preset from JSON: %s', $e->getMessage()), 1639146127, $e);
+        }
+        $instance = new self($data['synchronizeResources']);
+        if (isset($data['assetSourceIdentifiers'])) {
+            $instance->assetSourceIdentifiers = $data['assetSourceIdentifiers'];
+        }
+        return $instance;
+    }
+
+    public function filterAssetSourceIdentifiers(array $assertSourceIdentifiers): self
+    {
+        $newInstance = clone $this;
+        $newInstance->assetSourceIdentifiers = $assertSourceIdentifiers;
+        return $newInstance;
+    }
+
+    public function hasAssetSourceFilter(): bool
+    {
+        return $this->assetSourceIdentifiers !== [];
+    }
+
+    public function toJson(): string
+    {
+        try {
+            return json_encode($this, JSON_THROW_ON_ERROR);
+        } catch (\JsonException $e) {
+            throw new \RuntimeException(sprintf('Failed to encode preset to JSON: %s', $e->getMessage()), 1639146100, $e);
+        }
+    }
+
+    public function jsonSerialize(): array
+    {
+        $data = [
+            'synchronizeResources' => $this->synchronizeResources
+        ];
+        if ($this->hasAssetSourceFilter()) {
+            $data['assetSourceIdentifiers'] = $this->assetSourceIdentifiers;
+        }
+        return $data;
+    }
+}

Configuration/Objects.yaml

@@ -0,0 +1,2 @@
+Wwwision\AssetSync\SyncService:
+  scope: singleton

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Bastian Waidelich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.