diff --git a/_config.php b/_config.php index 56fb6865..96cc4259 100644 --- a/_config.php +++ b/_config.php @@ -10,4 +10,4 @@ // Shortcode parser which only regenerates shortcodes ShortcodeParser::get('regenerator') - ->register('image', [ImageShortcodeProvider::class, 'regenerate_shortcode']); \ No newline at end of file + ->register('image', [ImageShortcodeProvider::class, 'regenerate_shortcode']); diff --git a/src/Dev/Tasks/FileMigrationHelper.php b/src/Dev/Tasks/FileMigrationHelper.php new file mode 100644 index 00000000..fbff35ef --- /dev/null +++ b/src/Dev/Tasks/FileMigrationHelper.php @@ -0,0 +1,356 @@ + '%$' . LoggerInterface::class, + ]; + + /** @var LoggerInterface */ + private $logger; + + /** @var FlysystemAssetStore */ + private $store; + + /** + * If a file fails to validate during migration, delete it. + * If set to false, the record will exist but will not be attached to any filesystem + * item anymore. + * + * @config + * @var bool + */ + private static $delete_invalid_files = true; + + /** @var int[] List of files IDs to delete */ + protected $legacyFileIDsToDelete = []; + + + public function __construct() + { + $this->logger = new NullLogger(); + } + + /** + * Perform migration + * + * @param string $base Absolute base path (parent of assets folder). Will default to PUBLIC_PATH + * @return int Number of files successfully migrated + */ + public function run($base = null) + { + $this->store = Injector::inst()->get(AssetStore::class); + if (!$this->store instanceof AssetStore || !method_exists($this->store, 'normalisePath')) { + throw new LogicException( + 'FileMigrationHelper: Can not run if the default asset store does not have a `normalisePath` method.' + ); + } + + if (empty($base)) { + $base = PUBLIC_PATH; + } + + // Set max time and memory limit + Environment::increaseTimeLimitTo(); + Environment::increaseMemoryLimitTo(); + + $this->logger->info('MIGRATING SILVERSTRIPE 3 LEGACY FILES'); + $ss3Count = $this->ss3Migration($base); + + $this->logger->info('NORMALISE SILVERTSTRIPE 4 FILES'); + $ss4Count = 0; + if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) { + $this->logger->info('Looking at live files'); + $ss4Count += Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::LIVE); + return $this->normaliseAllFiles('on the live stage'); + }); + + $this->logger->info('Looking at draft files'); + $ss4Count += Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::DRAFT); + return $this->normaliseAllFiles('on the draft stage'); + }); + } else { + $ss4Count = $this->normaliseAllFiles(''); + } + + if ($ss4Count > 0) { + $this->logger->info(sprintf('%d files were normalised', $ss4Count)); + } else { + $this->logger->info('No files needed to be normalised'); + } + + return $ss3Count + $ss4Count; + } + + protected function ss3Migration($base) + { + // Check if the File dataobject has a "Filename" field. + // If not, cannot migrate + /** @skipUpgrade */ + if (!DB::get_schema()->hasField('File', 'Filename')) { + return 0; + } + + // Clean up SS3 files + $ss3Count = 0; + $originalState = null; + if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) { + $originalState = Versioned::get_reading_mode(); + Versioned::set_stage(Versioned::DRAFT); + } + + foreach ($this->getLegacyFileQuery() as $file) { + // Bypass the accessor and the filename from the column + $filename = $file->getField('Filename'); + + $success = $this->migrateFile($base, $file, $filename); + if ($success) { + $ss3Count++; + } + } + + $ss3InvalidCount = $this->deleteInvalidSilverStripeThreeFiles(); + + // Show summary of results + if ($ss3Count > 0) { + $this->logger->info(sprintf('%d legacy files have been migrated.', $ss3Count)); + } else { + $this->logger->info(sprintf('No SilverStripe 3 files have been migrated.', $ss3Count)); + } + + if ($ss3InvalidCount > 0) { + $this->logger->info( + sprintf('%d invalid SilverStripe 3 have been deleted from the DB.', $ss3InvalidCount) + ); + } + + if (class_exists(Versioned::class)) { + Versioned::set_reading_mode($originalState); + } + + return $ss3Count; + } + + /** + * Migrate a single file + * + * @param string $base Absolute base path (parent of assets folder) + * @param File $file + * @param string $legacyFilename + * @return bool True if this file is imported successfully + */ + protected function migrateFile($base, File $file, $legacyFilename) + { + // Make sure this legacy file actually exists + $path = $base . '/' . $legacyFilename; + if (!file_exists($path)) { + return false; + } + + // Remove this file if it violates allowed_extensions + $allowed = array_filter(File::getAllowedExtensions()); + $extension = strtolower($file->getExtension()); + if (!in_array($extension, $allowed)) { + if ($this->config()->get('delete_invalid_files')) { + // We're chunking out queries, we don't want to delete our entry right away, because that would + // alter the order of our results. + $this->legacyFileIDsToDelete[] = $file->ID; + } + return false; + } + + // Fix file classname if it has a classname that's incompatible with it's extention + if (!$this->validateFileClassname($file, $extension)) { + // We disable validation (if it is enabled) so that we are able to write a corrected + // classname, once that is changed we re-enable it for subsequent writes + $validationEnabled = DataObject::Config()->get('validation_enabled'); + if ($validationEnabled) { + DataObject::Config()->set('validation_enabled', false); + } + $destinationClass = $file->get_class_for_file_extension($extension); + $file = $file->newClassInstance($destinationClass); + $fileID = $file->write(); + if ($validationEnabled) { + DataObject::Config()->set('validation_enabled', true); + } + $file = File::get_by_id($fileID); + } + + // Copy local file into this filesystem + $filename = $file->generateFilename(); + $results = $this->store->normalisePath($filename); + + // Move file if the APL changes filename value + $file->File->Filename = $results['Filename']; + $file->File->Hash = $results['Hash']; + + + // Save and publish + $file->write(); + if (class_exists(Versioned::class)) { + $file->copyVersionToStage(Versioned::DRAFT, Versioned::LIVE); + } + + $this->logger->info(sprintf('* SS3 file %s converted to SS4 format', $file->getFilename())); + if (!empty($results['Operations'])) { + foreach ($results['Operations'] as $origin => $destination) { + $this->logger->info(sprintf(' * %s moved to %s', $origin, $destination)); + } + } + + return true; + } + + /** + * Delete all the invalid SS3 files after we've run `migrateFile`. + */ + private function deleteInvalidSilverStripeThreeFiles() + { + $count = sizeof($this->legacyFileIDsToDelete); + + $chunks = array_chunk($this->legacyFileIDsToDelete, 100); + $this->legacyFileIDsToDelete = []; + + // Unfortunately we can't run a query directly against the Files table, because it's versioned. We need to go + // through the ORM. + while ($chunkOfIDs = array_shift($chunks)) { + $files = File::get()->filter('ID', $chunkOfIDs); + foreach ($files as $file) { + $file->delete(); + } + } + + return $count; + } + + protected function normaliseAllFiles($stageString) + { + $count = 0; + + $files = $this->chunk(File::get()->exclude('ClassName', [Folder::class, 'Folder'])); + + /** @var File $file */ + foreach ($files as $file) { + $results = $this->store->normalise($file->File->Filename, $file->File->Hash); + if ($results && !empty($results['Operations'])) { + $this->logger->info( + sprintf('* %s has been normalised %s', $file->getFilename(), $stageString) + ); + foreach ($results['Operations'] as $origin => $destination) { + $this->logger->info(sprintf(' * %s moved to %s', $origin, $destination)); + } + $count++; + } + } + + return $count; + } + + /** + * Check if a file's classname is compatible with it's extension + * + * @param File $file + * @param string $extension + * @return bool + */ + protected function validateFileClassname($file, $extension) + { + $destinationClass = $file->get_class_for_file_extension($extension); + return $file->ClassName === $destinationClass; + } + + /** + * Get list of File dataobjects to import + * + * @return DataList + */ + protected function getFileQuery() + { + $table = DataObject::singleton(File::class)->baseTable(); + // Select all records which have a Filename value, but not FileFilename. + /** @skipUpgrade */ + return File::get() + ->exclude('ClassName', [Folder::class, 'Folder']) + ->filter('FileFilename', array('', null)) + ->sort('ID') + ->where(sprintf( + '"%s"."Filename" IS NOT NULL AND "%s"."Filename" != \'\'', + $table, + $table + )) // Non-orm field + ->alterDataQuery(function (DataQuery $query) use ($table) { + return $query->addSelectFromTable($table, ['Filename']); + }); + } + + protected function getLegacyFileQuery() + { + return $this->chunk($this->getFileQuery()); + } + + private function chunk(DataList $query) + { + $chunkSize = 100; + $page = 0; + while ($chunk = $query->limit($chunkSize, $chunkSize * $page)->getIterator()) { + foreach ($chunk as $file) { + yield $file; + } + + if ($chunk->count() < $chunkSize) { + break; + } + + $page++; + } + } + + /** + * Get map of File IDs to legacy filenames + * + * @deprecated 4.4.0 + * @return array + */ + protected function getFilenameArray() + { + $table = DataObject::singleton(File::class)->baseTable(); + // Convert original query, ensuring the legacy "Filename" is included in the result + /** @skipUpgrade */ + return $this + ->getFileQuery() + ->dataQuery() + ->selectFromTable($table, ['ID', 'Filename']) + ->execute() + ->map(); // map ID to Filename + } +} diff --git a/src/FileMigrationHelper.php b/src/FileMigrationHelper.php index 55f1bb1b..ccdf4fe8 100644 --- a/src/FileMigrationHelper.php +++ b/src/FileMigrationHelper.php @@ -2,354 +2,13 @@ namespace SilverStripe\Assets; -use LogicException; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use SilverStripe\Assets\Flysystem\FlysystemAssetStore; -use SilverStripe\Assets\Storage\AssetStore; -use SilverStripe\Core\Config\Config; -use SilverStripe\Core\Config\Configurable; -use SilverStripe\Core\Environment; -use SilverStripe\Core\Injector\Injectable; -use SilverStripe\Core\Injector\Injector; -use SilverStripe\ORM\DataList; -use SilverStripe\ORM\DataObject; -use SilverStripe\ORM\DataQuery; -use SilverStripe\ORM\DB; -use SilverStripe\Versioned\Versioned; - /** * Service to help migrate File dataobjects to the new APL. * * This service does not alter these records in such a way that prevents downgrading back to 3.x + * + * @deprecated 1.4.0 Use \SilverStripe\Assets\Dev\Tasks\FileMigrationHelper instead */ -class FileMigrationHelper +class FileMigrationHelper extends \SilverStripe\Assets\Dev\Tasks\FileMigrationHelper { - use Injectable; - use Configurable; - - private static $dependencies = [ - 'logger' => '%$' . LoggerInterface::class, - ]; - - /** @var LoggerInterface */ - private $logger; - - /** @var FlysystemAssetStore */ - private $store; - - /** - * If a file fails to validate during migration, delete it. - * If set to false, the record will exist but will not be attached to any filesystem - * item anymore. - * - * @config - * @var bool - */ - private static $delete_invalid_files = true; - - /** @var int[] List of files IDs to delete */ - protected $legacyFileIDsToDelete = []; - - - public function __construct() - { - $this->logger = new NullLogger(); - } - - /** - * Perform migration - * - * @param string $base Absolute base path (parent of assets folder). Will default to PUBLIC_PATH - * @return int Number of files successfully migrated - */ - public function run($base = null) - { - $this->store = Injector::inst()->get(AssetStore::class); - if (!$this->store instanceof AssetStore || !method_exists($this->store, 'normalisePath')) { - throw new LogicException( - 'FileMigrationHelper: Can not run if the default asset store does not have a `normalisePath` method.' - ); - } - - if (empty($base)) { - $base = PUBLIC_PATH; - } - - // Set max time and memory limit - Environment::increaseTimeLimitTo(); - Environment::increaseMemoryLimitTo(); - - $this->logger->info('MIGRATING SILVERSTRIPE 3 LEGACY FILES'); - $ss3Count = $this->ss3Migration($base); - - $this->logger->info('NORMALISE SILVERTSTRIPE 4 FILES'); - $ss4Count = 0; - if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) { - $this->logger->info('Looking at live files'); - $ss4Count += Versioned::withVersionedMode(function() { - Versioned::set_stage(Versioned::LIVE); - return $this->normaliseAllFiles('on the live stage'); - }); - - $this->logger->info('Looking at draft files'); - $ss4Count += Versioned::withVersionedMode(function() { - Versioned::set_stage(Versioned::DRAFT); - return $this->normaliseAllFiles('on the draft stage'); - }); - } else { - $ss4Count = $this->normaliseAllFiles(''); - } - - if ($ss4Count > 0) { - $this->logger->info(sprintf('%d files were normalised', $ss4Count)); - } else { - $this->logger->info('No files needed to be normalised'); - } - - return $ss3Count + $ss4Count; - } - - protected function ss3Migration($base) - { - // Check if the File dataobject has a "Filename" field. - // If not, cannot migrate - /** @skipUpgrade */ - if (!DB::get_schema()->hasField('File', 'Filename')) { - return 0; - } - - // Clean up SS3 files - $ss3Count = 0; - $originalState = null; - if (class_exists(Versioned::class) && File::has_extension(Versioned::class)) { - $originalState = Versioned::get_reading_mode(); - Versioned::set_stage(Versioned::DRAFT); - } - - foreach ($this->getLegacyFileQuery() as $file) { - // Bypass the accessor and the filename from the column - $filename = $file->getField('Filename'); - - $success = $this->migrateFile($base, $file, $filename); - if ($success) { - $ss3Count++; - } - } - - $ss3InvalidCount = $this->deleteInvalidSilverStripeThreeFiles(); - - // Show summary of results - if ($ss3Count > 0) { - $this->logger->info(sprintf('%d legacy files have been migrated.', $ss3Count)); - } else { - $this->logger->info(sprintf('No SilverStripe 3 files have been migrated.', $ss3Count)); - } - - if ($ss3InvalidCount > 0) { - $this->logger->info( - sprintf('%d invalid SilverStripe 3 have been deleted from the DB.', $ss3InvalidCount) - ); - } - - if (class_exists(Versioned::class)) { - Versioned::set_reading_mode($originalState); - } - - return $ss3Count; - } - - /** - * Migrate a single file - * - * @param string $base Absolute base path (parent of assets folder) - * @param File $file - * @param string $legacyFilename - * @return bool True if this file is imported successfully - */ - protected function migrateFile($base, File $file, $legacyFilename) - { - // Make sure this legacy file actually exists - $path = $base . '/' . $legacyFilename; - if (!file_exists($path)) { - return false; - } - - // Remove this file if it violates allowed_extensions - $allowed = array_filter(File::getAllowedExtensions()); - $extension = strtolower($file->getExtension()); - if (!in_array($extension, $allowed)) { - if ($this->config()->get('delete_invalid_files')) { - // We're chunking out queries, we don't want to delete our entry right away, because that would - // alter the order of our results. - $this->legacyFileIDsToDelete[] = $file->ID; - } - return false; - } - - // Fix file classname if it has a classname that's incompatible with it's extention - if (!$this->validateFileClassname($file, $extension)) { - // We disable validation (if it is enabled) so that we are able to write a corrected - // classname, once that is changed we re-enable it for subsequent writes - $validationEnabled = DataObject::Config()->get('validation_enabled'); - if ($validationEnabled) { - DataObject::Config()->set('validation_enabled', false); - } - $destinationClass = $file->get_class_for_file_extension($extension); - $file = $file->newClassInstance($destinationClass); - $fileID = $file->write(); - if ($validationEnabled) { - DataObject::Config()->set('validation_enabled', true); - } - $file = File::get_by_id($fileID); - } - - // Copy local file into this filesystem - $filename = $file->generateFilename(); - $results = $this->store->normalisePath($filename); - - // Move file if the APL changes filename value - $file->File->Filename = $results['Filename']; - $file->File->Hash = $results['Hash']; - - - // Save and publish - $file->write(); - if (class_exists(Versioned::class)) { - $file->copyVersionToStage(Versioned::DRAFT, Versioned::LIVE); - } - - $this->logger->info(sprintf('* SS3 file %s converted to SS4 format', $file->getFilename())); - if (!empty($results['Operations'])) { - foreach ($results['Operations'] as $origin => $destination) { - $this->logger->info(sprintf(' * %s moved to %s', $origin, $destination)); - } - } - - return true; - } - - /** - * Delete all the invalid SS3 files after we've run `migrateFile`. - */ - private function deleteInvalidSilverStripeThreeFiles() - { - $count = sizeof($this->legacyFileIDsToDelete); - - $chunks = array_chunk($this->legacyFileIDsToDelete, 100); - $this->legacyFileIDsToDelete = []; - - // Unfortunately we can't run a query directly against the Files table, because it's versioned. We need to go - // through the ORM. - while ($chunkOfIDs = array_shift($chunks)) { - $files = File::get()->filter('ID', $chunkOfIDs); - foreach ($files as $file) { - $file->delete(); - } - } - - return $count; - } - - protected function normaliseAllFiles($stageString) - { - $count = 0; - - $files = $this->chunk(File::get()->exclude('ClassName', [Folder::class, 'Folder'])); - - /** @var File $file */ - foreach($files as $file) { - $results = $this->store->normalise($file->File->Filename, $file->File->Hash); - if ($results && !empty($results['Operations'])) { - $this->logger->info( - sprintf('* %s has been normalised %s', $file->getFilename(), $stage) - ); - foreach ($results['Operations'] as $origin => $destination) { - $this->logger->info(sprintf(' * %s moved to %s', $origin, $destination)); - } - $count++; - } - } - - return $count; - - } - - /** - * Check if a file's classname is compatible with it's extension - * - * @param File $file - * @param string $extension - * @return bool - */ - protected function validateFileClassname($file, $extension) - { - $destinationClass = $file->get_class_for_file_extension($extension); - return $file->ClassName === $destinationClass; - } - - /** - * Get list of File dataobjects to import - * - * @return DataList - */ - protected function getFileQuery() - { - $table = DataObject::singleton(File::class)->baseTable(); - // Select all records which have a Filename value, but not FileFilename. - /** @skipUpgrade */ - return File::get() - ->exclude('ClassName', [Folder::class, 'Folder']) - ->filter('FileFilename', array('', null)) - ->sort('ID') - ->where(sprintf( - '"%s"."Filename" IS NOT NULL AND "%s"."Filename" != \'\'', - $table, - $table - )) // Non-orm field - ->alterDataQuery(function (DataQuery $query) use ($table) { - return $query->addSelectFromTable($table, ['Filename']); - }); - } - - protected function getLegacyFileQuery() - { - return $this->chunk($this->getFileQuery()); - } - - private function chunk(DataList $query) - { - $chunkSize = 100; - $page = 0; - while ($chunk = $query->limit($chunkSize, $chunkSize * $page)->getIterator()) { - foreach ($chunk as $file) { - yield $file; - } - - if ($chunk->count() < $chunkSize) { - break; - } - - $page++; - } - } - - /** - * Get map of File IDs to legacy filenames - * - * @deprecated 4.4.0 - * @return array - */ - protected function getFilenameArray() - { - $table = DataObject::singleton(File::class)->baseTable(); - // Convert original query, ensuring the legacy "Filename" is included in the result - /** @skipUpgrade */ - return $this - ->getFileQuery() - ->dataQuery() - ->selectFromTable($table, ['ID', 'Filename']) - ->execute() - ->map(); // map ID to Filename - } } diff --git a/src/FilenameParsing/FileIDHelperResolutionStrategy.php b/src/FilenameParsing/FileIDHelperResolutionStrategy.php index 33c323f5..f1aa825b 100644 --- a/src/FilenameParsing/FileIDHelperResolutionStrategy.php +++ b/src/FilenameParsing/FileIDHelperResolutionStrategy.php @@ -370,20 +370,22 @@ public function findVariants($tuple, Filesystem $filesystem) $helpers = $this->getResolutionFileIDHelpers(); array_unshift($helpers, $this->getDefaultFileIDHelper()); + /** @var FileIDHelper[] $resolvableHelpers */ + $resolvableHelpers = []; + // Search for a helper that will allow us to find a file - /** @var FileIDHelper $helper */ - $helper = null; foreach ($helpers as $helper) { $fileID = $helper->buildFileID( $parsedFileID->getFilename(), $parsedFileID->getHash() ); - if (!$filesystem->has($fileID) || !$this->validateHash($helper, $parsedFileID, $filesystem)) { - // This helper didn't find our file - continue; + if ($filesystem->has($fileID) && $this->validateHash($helper, $parsedFileID, $filesystem)) { + $resolvableHelpers[] = $helper; } + } + foreach ($resolvableHelpers as $helper) { $folder = $helper->lookForVariantIn($parsedFileID); $possibleVariants = $filesystem->listContents($folder, true); foreach ($possibleVariants as $possibleVariant) { diff --git a/tests/php/FileMigrationHelperTest.php b/tests/php/Dev/Tasks/FileMigrationHelperTest.php similarity index 93% rename from tests/php/FileMigrationHelperTest.php rename to tests/php/Dev/Tasks/FileMigrationHelperTest.php index e4272a19..4875de84 100644 --- a/tests/php/FileMigrationHelperTest.php +++ b/tests/php/Dev/Tasks/FileMigrationHelperTest.php @@ -1,15 +1,15 @@ exclude('ClassName', Folder::class) as $file) { $dest = TestAssetStore::base_path() . '/' . $file->generateFilename(); Filesystem::makeFolder(dirname($dest)); @@ -58,7 +58,7 @@ public function setUp() } // Let's create some variants for our images - $from = __DIR__ . '/ImageTest/test-image-high-quality.jpg'; + $from = __DIR__ . '/../../ImageTest/test-image-high-quality.jpg'; foreach (Image::get() as $file) { $dest = TestAssetStore::base_path() . '/' . $file->generateFilename(); $dir = dirname($dest); @@ -84,9 +84,6 @@ public function tearDown() */ public function testMigration() { - // TODO Fix file migration test by adjusting file migration logic to new behaviour - // added through https://github.com/silverstripe/silverstripe-versioned/issues/177 - // Prior to migration, check that each file has empty Filename / Hash properties foreach (File::get()->exclude('ClassName', Folder::class) as $file) { $filename = $file->generateFilename(); diff --git a/tests/php/FileMigrationHelperTest.yml b/tests/php/Dev/Tasks/FileMigrationHelperTest.yml similarity index 100% rename from tests/php/FileMigrationHelperTest.yml rename to tests/php/Dev/Tasks/FileMigrationHelperTest.yml diff --git a/tests/php/FileMigrationHelperTest/Extension.php b/tests/php/Dev/Tasks/FileMigrationHelperTest/Extension.php similarity index 88% rename from tests/php/FileMigrationHelperTest/Extension.php rename to tests/php/Dev/Tasks/FileMigrationHelperTest/Extension.php index 32c00500..ae6ab591 100644 --- a/tests/php/FileMigrationHelperTest/Extension.php +++ b/tests/php/Dev/Tasks/FileMigrationHelperTest/Extension.php @@ -1,6 +1,6 @@ get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + $legacyHelper = new LegacyFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($legacyHelper); + $public->setResolutionFileIDHelpers([$legacyHelper]); + + $store->setPublicResolutionStrategy($public); + } + + protected function defineDestinationStrategy() + { + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + $naturalPath = new NaturalFileIDHelper(); + $legacyHelper = new LegacyFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($hashHelper); + $public->setResolutionFileIDHelpers([$hashHelper, $naturalPath, $legacyHelper]); + + $store->setPublicResolutionStrategy($public); + } +} diff --git a/tests/php/Dev/Tasks/SS4FileMigrationHelperTest.php b/tests/php/Dev/Tasks/SS4FileMigrationHelperTest.php new file mode 100644 index 00000000..06c05ee4 --- /dev/null +++ b/tests/php/Dev/Tasks/SS4FileMigrationHelperTest.php @@ -0,0 +1,310 @@ + array( + Extension::class, + ) + ); + + public function setUp() + { + Config::nest(); // additional nesting here necessary + Config::modify()->merge(File::class, 'migrate_legacy_file', false); + + // Set backend root to /FileMigrationHelperTest/assets + TestAssetStore::activate('FileMigrationHelperTest'); + $this->defineOriginStrategy(); + parent::setUp(); + + // Ensure that each file has a local record file in this new assets base + /** @var File $file */ + foreach (File::get()->filter('ClassName', File::class) as $file) { + $filename = $file->getFilename(); + + // Create an archive version of the file + DBDatetime::set_mock_now('2000-01-01 11:00:00'); + $file->setFromString('Archived content of ' . $filename, $filename); + $file->write(); + $file->publishSingle(); + DBDatetime::clear_mock_now(); + + // Publish a version of the file + $file->setFromString('Published content of ' . $filename, $filename); + $file->write(); + $file->publishSingle(Versioned::DRAFT, Versioned::LIVE); + + // Create a draft of the file + $file->setFromString('Draft content of ' . $filename, $filename); + $file->write(); + } + + // Let's create some variants for our images + /** @var Image $image */ + foreach (Image::get() as $image) { + $filename = $image->getFilename(); + + // Create an archive version of our image with a thumbnail + DBDatetime::set_mock_now('2000-01-01 11:00:00'); + $stream = $this->generateImage('Archived', $filename)->stream($image->getExtension()); + $image->setFromStream(StreamWrapper::getResource($stream), $filename); + $image->write(); + $image->CMSThumbnail(); + $image->publishSingle(); + DBDatetime::clear_mock_now(); + + // Publish a live version of our image with a thumbnail + $stream = $this->generateImage('Published', $filename)->stream($image->getExtension()); + $image->setFromStream(StreamWrapper::getResource($stream), $filename); + $image->write(); + $image->CMSThumbnail(); + $image->publishSingle(); + + // Create a draft version of our images with a thumbnail + $stream = $this->generateImage('Draft', $filename)->stream($image->getExtension()); + $image->setFromStream(StreamWrapper::getResource($stream), $filename); + $image->CMSThumbnail(); + $image->write(); + } + + $this->defineDestinationStrategy(); + } + + /** + * Generate a placeholder image + * @param string $targetedStage + * @param string $filename + * @return \Intervention\Image\Image + */ + private function generateImage($targetedStage, $filename) + { + /** @var ImageManager $imageManager */ + $imageManager = Injector::inst()->create(ImageManager::class); + return $imageManager + ->canvas(400, 300, '#142237') + ->text($targetedStage, 20, 170, function (AbstractFont $font) { + $font->color('#44C8F5'); + })->text($filename, 20, 185, function (AbstractFont $font) { + $font->color('#ffffff'); + })->rectangle(20, 200, 100, 202, function (AbstractShape $shape) { + $shape->background('#DA1052'); + }); + } + + /** + * Called by set up before creating all the fixture entries. Defines the original startegies for the assets store. + */ + protected function defineOriginStrategy() + { + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($hashHelper); + $public->setResolutionFileIDHelpers([$hashHelper]); + + $store->setPublicResolutionStrategy($public); + } + + /** + * Called by set up after creating all the fixture entries. Defines the targeted strategies that the + * FileMigrationHelper should move the files to. + */ + protected function defineDestinationStrategy() + { + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + $hashHelper = new HashFileIDHelper(); + $naturalHelper = new NaturalFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($hashHelper); + $protected->setResolutionFileIDHelpers([$hashHelper, $naturalHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($naturalHelper); + $public->setResolutionFileIDHelpers([$hashHelper, $naturalHelper]); + + $store->setPublicResolutionStrategy($public); + } + + public function tearDown() + { + TestAssetStore::reset(); + parent::tearDown(); + Config::unnest(); + } + + public function testMigration() + { + $helper = new FileMigrationHelper(); + $result = $helper->run(TestAssetStore::base_path()); + + // Let's look at our draft files + Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::DRAFT); + foreach (File::get()->filter('ClassName', File::class) as $file) { + $this->assertFileAt($file, AssetStore::VISIBILITY_PROTECTED, 'Draft'); + } + + foreach (Image::get() as $image) { + $this->assertImageAt($image, AssetStore::VISIBILITY_PROTECTED, 'Draft'); + } + }); + + // Let's look at our live files + Versioned::withVersionedMode(function () { + Versioned::set_stage(Versioned::LIVE); + + // There's one file with restricted views, the published version of this file will be protected + $restrictedFileID = $this->idFromFixture(File::class, 'restrictedViewFolder-file4'); + $this->lookAtRestrictedFile($restrictedFileID); + + /** @var File $file */ + foreach (File::get()->filter('ClassName', File::class)->exclude('ID', $restrictedFileID) as $file) { + $this->assertFileAt($file, AssetStore::VISIBILITY_PUBLIC, 'Published'); + } + + foreach (Image::get() as $image) { + $this->assertImageAt($image, AssetStore::VISIBILITY_PUBLIC, 'Published'); + } + }); + } + + /** + * Test that this restricted file is protected. This test is in its own method so that transition where this + * scenario can not exist can override it. + * @param $restrictedFileID + */ + protected function lookAtRestrictedFile($restrictedFileID) + { + $restrictedFile = File::get()->byID($restrictedFileID); + $this->assertFileAt($restrictedFile, AssetStore::VISIBILITY_PROTECTED, 'Published'); + } + + /** + * Convenience method to group a bunch of assertions about a regular files + * @param File $file + * @param string $visibility Expected visibility of the file + * @param string $stage Stage that we are testing, will appear in some error messages and in the expected content + */ + protected function assertFileAt(File $file, $visibility, $stage) + { + $ucVisibility = ucfirst($visibility); + $filename = $file->getFilename(); + $hash = $file->getHash(); + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + /** @var Filesystem $fs */ + $fs = call_user_func([$store, "get{$ucVisibility}Filesystem"]); + /** @var FileResolutionStrategy $strategy */ + $strategy = call_user_func([$store, "get{$ucVisibility}ResolutionStrategy"]); + + $this->assertEquals( + $visibility, + $store->getVisibility($filename, $hash), + sprintf('%s version of %s should be %s', $stage, $filename, $visibility) + ); + $expectedURL = $strategy->buildFileID(new ParsedFileID($filename, $hash)); + $this->assertTrue( + $fs->has($expectedURL), + sprintf('%s version of %s should be on %s store under %s', $stage, $filename, $visibility, $expectedURL) + ); + $this->assertEquals( + sprintf('%s content of %s', $stage, $filename), + $fs->read($expectedURL), + sprintf('%s version of %s on %s store has wrong content', $stage, $filename, $visibility) + ); + } + + /** + * Convenience method to group a bunch of assertions about an image + * @param File $file + * @param string $visibility Expected visibility of the file + * @param string $stage Stage that we are testing, will appear in some error messages + */ + protected function assertImageAt(Image $file, $visibility, $stage) + { + $ucVisibility = ucfirst($visibility); + $filename = $file->getFilename(); + $hash = $file->getHash(); + $pfid = new ParsedFileID($filename, $hash); + + /** @var FlysystemAssetStore $store */ + $store = Injector::inst()->get(AssetStore::class); + + /** @var Filesystem $fs */ + $fs = call_user_func([$store, "get{$ucVisibility}Filesystem"]); + /** @var FileResolutionStrategy $strategy */ + $strategy = call_user_func([$store, "get{$ucVisibility}ResolutionStrategy"]); + + $this->assertEquals( + $visibility, + $store->getVisibility($filename, $hash), + sprintf('%s version of %s should be %s', $stage, $filename, $visibility) + ); + + $expectedURL = $strategy->buildFileID($pfid); + $this->assertTrue( + $fs->has($expectedURL), + sprintf('%s version of %s should be on %s store under %s', $stage, $filename, $visibility, $expectedURL) + ); + $expectedURL = $strategy->buildFileID($pfid->setVariant('FillWzEwMCwxMDBd')); + $this->assertTrue( + $fs->has($expectedURL), + sprintf('%s thumbnail of %s should be on %s store under %s', $stage, $filename, $visibility, $expectedURL) + ); + } +} diff --git a/tests/php/Dev/Tasks/SS4KeepArchivedFileMigrationHelperTest.php b/tests/php/Dev/Tasks/SS4KeepArchivedFileMigrationHelperTest.php new file mode 100644 index 00000000..f83bda21 --- /dev/null +++ b/tests/php/Dev/Tasks/SS4KeepArchivedFileMigrationHelperTest.php @@ -0,0 +1,43 @@ +filter('ClassName', File::class) as $file) { + $this->assertFileAt($file, AssetStore::VISIBILITY_PROTECTED, 'Archived'); + } + + foreach (Image::get() as $image) { + $this->assertImageAt($image, AssetStore::VISIBILITY_PROTECTED, 'Archived'); + } + }); + } + + protected function defineOriginStrategy() + { + parent::defineOriginStrategy(); + + File::config()->set('keep_archived_assets', true); + } +} diff --git a/tests/php/Dev/Tasks/SS4LegacyFileMigrationHelperTest.php b/tests/php/Dev/Tasks/SS4LegacyFileMigrationHelperTest.php new file mode 100644 index 00000000..abaa3b66 --- /dev/null +++ b/tests/php/Dev/Tasks/SS4LegacyFileMigrationHelperTest.php @@ -0,0 +1,45 @@ +get(AssetStore::class); + + $naturalHelper = new NaturalFileIDHelper(); + + $protected = new FileIDHelperResolutionStrategy(); + $protected->setVersionedStage(Versioned::DRAFT); + $protected->setDefaultFileIDHelper($naturalHelper); + $protected->setResolutionFileIDHelpers([$naturalHelper]); + + $store->setProtectedResolutionStrategy($protected); + + $public = new FileIDHelperResolutionStrategy(); + $public->setVersionedStage(Versioned::LIVE); + $public->setDefaultFileIDHelper($naturalHelper); + $public->setResolutionFileIDHelpers([$naturalHelper]); + + $store->setPublicResolutionStrategy($public); + } + + protected function lookAtRestrictedFile($restrictedFileID) + { + // Legacy files names did not allow you to have a restricted file in draft and live simultanously + } +} diff --git a/tests/php/FileTest.yml b/tests/php/FileTest.yml index 15f89a46..b635ddc3 100644 --- a/tests/php/FileTest.yml +++ b/tests/php/FileTest.yml @@ -54,11 +54,11 @@ SilverStripe\Assets\Folder: Name: FileTest-folder1-subfolder1 ParentID: =>SilverStripe\Assets\Folder.folder1 restrictedFolder: - Name: FileTest-restricted-folder + Name: restricted-folder CanEditType: OnlyTheseUsers EditorGroups: =>SilverStripe\Security\Group.assetusers restrictedViewFolder: - Name: FileTest-restricted-view-folder + Name: restricted-view-folder CanViewType: OnlyTheseUsers ViewerGroups: =>SilverStripe\Security\Group.assetusers deep-folder: @@ -89,12 +89,12 @@ SilverStripe\Assets\File: Name: File1.txt ParentID: =>SilverStripe\Assets\Folder.folder1 restrictedFolder-file3: - FileFilename: restrictedFolder/File3.txt + FileFilename: restricted-folder/File3.txt FileHash: 55b443b60176235ef09801153cca4e6da7494a0c - Name: File1.txt + Name: File3.txt ParentID: =>SilverStripe\Assets\Folder.restrictedFolder restrictedViewFolder-file4: - FileFilename: restrictedViewFolder/File4.txt + FileFilename: restricted-view-folder/File4.txt FileHash: 55b443b60176235ef09801153cca4e6da7494a0c Name: File4.txt ParentID: =>SilverStripe\Assets\Folder.restrictedViewFolder