diff --git a/modules/common/tests/src/Kernel/ConfigFormTestBase.php b/modules/common/tests/src/Kernel/ConfigFormTestBase.php index f4c40163a2..96e18fa087 100644 --- a/modules/common/tests/src/Kernel/ConfigFormTestBase.php +++ b/modules/common/tests/src/Kernel/ConfigFormTestBase.php @@ -38,7 +38,7 @@ abstract class ConfigFormTestBase extends KernelTestBase { * ); * @endcode * - * @return array[] + * @return array[] * Form test data. */ abstract public function provideFormData(): array; @@ -69,7 +69,7 @@ public function testConfigForm(array $form_values) { $this->assertTrue($valid_form, new FormattableMarkup('Input values: %values
Validation handler errors: %errors', $args)); foreach ($form_values as $data) { - $this->assertEquals($this->config($data['#config_name'])->get($data['#config_key']), $data['#value']); + $this->assertEquals($data['#value'], $this->config($data['#config_name'])->get($data['#config_key'])); } } diff --git a/modules/datastore/datastore.services.yml b/modules/datastore/datastore.services.yml index 9bd513d78a..4d6b5a453f 100644 --- a/modules/datastore/datastore.services.yml +++ b/modules/datastore/datastore.services.yml @@ -21,6 +21,20 @@ services: - '@dkan.common.drupal_files' - '@dkan.common.job_store' + dkan.datastore.service.resource_processor_collector: + class: \Drupal\datastore\Service\ResourceProcessorCollector + tags: + - { name: service_collector, tag: resource_processor, call: addResourceProcessor } + + dkan.datastore.service.resource_processor.dictionary_enforcer: + class: \Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer + arguments: + - '@dkan.datastore.data_dictionary.alter_table_query_factory.mysql' + - '@dkan.metastore.service' + - '@dkan.metastore.data_dictionary_discovery' + tags: + - { name: resource_processor, priority: 25 } + dkan.datastore.service.resource_purger: class: \Drupal\datastore\Service\ResourcePurger arguments: diff --git a/modules/datastore/src/Plugin/QueueWorker/DictionaryEnforcer.php b/modules/datastore/src/Plugin/QueueWorker/DictionaryEnforcer.php deleted file mode 100644 index 79ab8ad85a..0000000000 --- a/modules/datastore/src/Plugin/QueueWorker/DictionaryEnforcer.php +++ /dev/null @@ -1,205 +0,0 @@ -logger = $logger_factory->get('datastore'); - $this->metastore = $metastore; - // Set the timeout for database connections to the queue lease time. - // This ensures that database connections will remain open for the - // duration of the time the queue is being processed. - $timeout = (int) $plugin_definition['cron']['lease_time']; - $this->alterTableQueryFactory = $alter_table_query_factory->setConnectionTimeout($timeout); - $this->dataDictionaryDiscovery = $data_dictionary_discovery; - $this->resourceMapper = $resource_mapper; - } - - /** - * {@inheritdoc} - */ - public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { - return new static( - $configuration, - $plugin_id, - $plugin_definition, - $container->get('dkan.datastore.data_dictionary.alter_table_query_factory.mysql'), - $container->get('logger.factory'), - $container->get('dkan.metastore.service'), - $container->get('dkan.metastore.data_dictionary_discovery'), - $container->get('dkan.metastore.resource_mapper') - ); - } - - /** - * {@inheritdoc} - */ - public function processItem($data) { - // Catch and log any exceptions thrown when processing the queue item to - // prevent the item from being requeued. - try { - $this->doProcessItem($data); - } - catch (\Exception $e) { - $this->logger->error($e->getMessage()); - } - } - - /** - * Retrieve dictionary and datastore table details; apply dictionary to table. - * - * @param \Drupal\common\Resource $resource - * DKAN Resource. - */ - public function doProcessItem(Resource $resource): void { - $identifier = $resource->getIdentifier(); - $version = $resource->getVersion(); - - $latest_resource = $this->resourceMapper->get($identifier); - // Do not apply data-dictionary if resource no longer exists. - if (!isset($latest_resource)) { - $this->logger->notice('Cancelling data-dictionary enforcement; resource no longer exists.'); - return; - } - // Do not apply data-dictionary if resource has changed. - if ($version !== $latest_resource->getVersion()) { - $this->logger->notice('Cancelling data-dictionary enforcement; resource has changed.'); - return; - } - - // Retrieve name of datastore table for resource. - $datastore_table = $resource->getTableName(); - // Get data-dictionary for the given resource. - $dictionary = $this->getDataDictionaryForResource($resource); - // Extract data-dictionary field types. - $dictionary_fields = $dictionary->{'$.data.fields'}; - - $this->applyDictionary($dictionary_fields, $datastore_table); - } - - /** - * Retrieve the data-dictionary metadata object for the given resource. - * - * @param \Drupal\common\Resource $resource - * DKAN Resource. - * - * @return \RootedData\RootedJsonData - * Data-dictionary metadata. - */ - protected function getDataDictionaryForResource(Resource $resource): RootedJsonData { - $resource_id = $resource->getIdentifier(); - $resource_version = $resource->getVersion(); - $dict_id = $this->dataDictionaryDiscovery->dictionaryIdFromResource($resource_id, $resource_version); - - if (!isset($dict_id)) { - throw new \UnexpectedValueException(sprintf('No data-dictionary found for resource with id "%s" and version "%s".', $resource_id, $resource_version)); - } - return $this->metastore->get('data-dictionary', $dict_id); - } - - /** - * Apply data types in the given dictionary fields to the given datastore. - * - * @param array $dictionary_fields - * Data dictionary fields. - * @param string $datastore_table - * Mysql table name. - */ - public function applyDictionary(array $dictionary_fields, string $datastore_table): void { - $this->alterTableQueryFactory - ->getQuery($datastore_table, $dictionary_fields) - ->applyDataTypes(); - } - -} diff --git a/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php b/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php new file mode 100644 index 0000000000..74452d7d91 --- /dev/null +++ b/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php @@ -0,0 +1,145 @@ +logger = $logger_factory->get('datastore'); + $this->resourceMapper = $resource_mapper; + $this->resourceProcessorCollector = $processor_collector; + // Set the timeout for database connections to the queue lease time. + // This ensures that database connections will remain open for the + // duration of the time the queue is being processed. + $timeout = (int) $plugin_definition['cron']['lease_time']; + $alter_table_query_factory->setConnectionTimeout($timeout); + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + return new static( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('dkan.datastore.data_dictionary.alter_table_query_factory.mysql'), + $container->get('logger.factory'), + $container->get('dkan.metastore.resource_mapper'), + $container->get('dkan.datastore.service.resource_processor_collector'), + ); + } + + /** + * {@inheritdoc} + */ + public function processItem($data) { + // Catch and log any exceptions thrown when processing the queue item to + // prevent the item from being requeued. + try { + $this->doProcessItem($data); + } + catch (\Exception $e) { + $this->logger->error($e->getMessage()); + } + } + + /** + * Pass along new resource to resource processors. + * + * @param \Drupal\common\Resource $resource + * DKAN Resource. + */ + public function doProcessItem(Resource $resource): void { + $identifier = $resource->getIdentifier(); + $version = $resource->getVersion(); + + $latest_resource = $this->resourceMapper->get($identifier); + // Stop if resource no longer exists. + if (!isset($latest_resource)) { + $this->logger->notice('Cancelling resource processing; resource no longer exists.'); + return; + } + // Stop if resource has changed. + if ($version !== $latest_resource->getVersion()) { + $this->logger->notice('Cancelling resource processing; resource has changed.'); + return; + } + // Run all tagged resource processors. + $processors = $this->resourceProcessorCollector->getResourceProcessors(); + array_map(fn ($processor) => $processor->process($resource), $processors); + } + +} diff --git a/modules/datastore/src/Service/Import.php b/modules/datastore/src/Service/Import.php index 1fae1dea0b..c9a31d7879 100644 --- a/modules/datastore/src/Service/Import.php +++ b/modules/datastore/src/Service/Import.php @@ -10,7 +10,6 @@ use Drupal\common\Storage\JobStoreFactory; use Drupal\datastore\Storage\DatabaseTable; use Drupal\datastore\Storage\DatabaseTableFactory; -use Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface; use Procrastinator\Result; /** @@ -117,12 +116,9 @@ public function import() { } // If the import job finished successfully... elseif ($result->getStatus() === Result::DONE) { - $dd_discovery = \Drupal::service('dkan.metastore.data_dictionary_discovery'); - if ($dd_discovery->getDataDictionaryMode() !== DataDictionaryDiscoveryInterface::MODE_NONE) { - // Queue the imported resource for data-dictionary enforcement. - $dictionary_enforcer_queue = \Drupal::service('queue')->get('dictionary_enforcer'); - $dictionary_enforcer_queue->createItem($resource); - } + // Queue the imported resource for post-import processing. + $post_import_queue = \Drupal::service('queue')->get('post_import'); + $post_import_queue->createItem($resource); } } diff --git a/modules/datastore/src/Service/ResourceProcessor/DictionaryEnforcer.php b/modules/datastore/src/Service/ResourceProcessor/DictionaryEnforcer.php new file mode 100644 index 0000000000..406fb2b72b --- /dev/null +++ b/modules/datastore/src/Service/ResourceProcessor/DictionaryEnforcer.php @@ -0,0 +1,117 @@ +metastore = $metastore; + $this->dataDictionaryDiscovery = $data_dictionary_discovery; + $this->alterTableQueryFactory = $alter_table_query_factory; + } + + /** + * Retrieve dictionary and datastore table details; apply dictionary to table. + * + * @param \Drupal\common\Resource $resource + * DKAN Resource. + */ + public function process(Resource $resource): void { + // Get data-dictionary for the given resource. + $dictionary = $this->getDataDictionaryForResource($resource); + // Extract data-dictionary field types. + $dictionary_fields = $dictionary->{'$.data.fields'}; + // Retrieve name of datastore table for resource. + $datastore_table = $resource->getTableName(); + + $this->applyDictionary($dictionary_fields, $datastore_table); + } + + /** + * Retrieve the data-dictionary metadata object for the given resource. + * + * @param \Drupal\common\Resource $resource + * DKAN Resource. + * + * @return \RootedData\RootedJsonData + * Data-dictionary metadata. + */ + protected function getDataDictionaryForResource(Resource $resource): RootedJsonData { + $resource_id = $resource->getIdentifier(); + $resource_version = $resource->getVersion(); + $dict_id = $this->dataDictionaryDiscovery->dictionaryIdFromResource($resource_id, $resource_version); + + if (!isset($dict_id)) { + throw new \UnexpectedValueException(sprintf('No data-dictionary found for resource with id "%s" and version "%s".', $resource_id, $resource_version)); + } + return $this->metastore->get('data-dictionary', $dict_id); + } + + /** + * Apply data types in the given dictionary fields to the given datastore. + * + * @param array $dictionary_fields + * Data dictionary fields. + * @param string $datastore_table + * Mysql table name. + */ + public function applyDictionary(array $dictionary_fields, string $datastore_table): void { + $this->alterTableQueryFactory + ->getQuery($datastore_table, $dictionary_fields) + ->applyDataTypes(); + } + +} diff --git a/modules/datastore/src/Service/ResourceProcessorCollector.php b/modules/datastore/src/Service/ResourceProcessorCollector.php new file mode 100644 index 0000000000..8174f78a4c --- /dev/null +++ b/modules/datastore/src/Service/ResourceProcessorCollector.php @@ -0,0 +1,42 @@ +processors[$priority] = $processor; + } + + /** + * Retrieve collected resource processors. + * + * @return \Drupal\datastore\Service\ResourceProcessorInterface[] + * Collected resource processors sorted in ascending order of priority. + */ + public function getResourceProcessors(): array { + // Sort processors by priority. + ksort($this->processors); + + return $this->processors; + } + +} diff --git a/modules/datastore/src/Service/ResourceProcessorInterface.php b/modules/datastore/src/Service/ResourceProcessorInterface.php new file mode 100644 index 0000000000..d4bfa10eda --- /dev/null +++ b/modules/datastore/src/Service/ResourceProcessorInterface.php @@ -0,0 +1,20 @@ +getContainerChain($resource->getVersion()) - ->add(AlterTableQueryInterface::class, 'applyDataTypes'); - \Drupal::setContainer($containerChain->getMock($resource->getVersion())); - - $dictionaryEnforcer = DictionaryEnforcer::create( - $containerChain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $dictionaryEnforcer->processItem($resource); - - // Assert no exceptions are thrown in happy path. - $errors = $containerChain->getStoredInput('error'); - $this->assertEmpty($errors); - } - - /** - * Test exception thrown in applyDataTypes() is caught and logged. - */ - public function testProcessItemApplyDataTypesException() { - $resource = new Resource('test.csv', 'text/csv'); - - $errorMessage = "Something went wrong: " . uniqid(); - - $containerChain = $this->getContainerChain($resource->getVersion()) - ->add(AlterTableQueryInterface::class, 'applyDataTypes', new \Exception($errorMessage)); - - $dictionaryEnforcer = DictionaryEnforcer::create( - $containerChain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $dictionaryEnforcer->processItem($resource); - - // Assert the log contains the expected exception message thrown earlier. - $this->assertEquals($errorMessage, $containerChain->getStoredInput('error')[0]); - } - - /** - * Test exception thrown in applyDataTypes() is caught and logged. - */ - public function testProcessItemUnableToFindDataDictionaryForResourceException() { - $resource = new Resource('test.csv', 'text/csv'); - - $containerChain = $this->getContainerChain($resource->getVersion()) - ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', NULL); - - $dictionaryEnforcer = DictionaryEnforcer::create( - $containerChain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $dictionaryEnforcer->processItem($resource); - $this->assertEquals( - sprintf('No data-dictionary found for resource with id "%s" and version "%s".', $resource->getIdentifier(), $resource->getVersion()), - $containerChain->getStoredInput('error')[0] - ); - } - - /** - * Get container chain. - */ - private function getContainerChain(int $resource_version) { - - $options = (new Options()) - ->add('dkan.datastore.data_dictionary.alter_table_query_factory.mysql', AlterTableQueryFactoryInterface::class) - ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscovery::class) - ->add('logger.factory', LoggerChannelFactoryInterface::class) - ->add('dkan.metastore.service', MetastoreService::class) - ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscoveryInterface::class) - ->add('stream_wrapper_manager', StreamWrapperManager::class) - ->add('dkan.metastore.resource_mapper', ResourceMapper::class) - ->index(0); - - $json = '{"identifier":"foo","title":"bar","data":{"fields":[]}}'; - - return (new Chain($this)) - ->add(Container::class, 'get', $options) - ->add(LoggerChannelFactoryInterface::class, 'get', LoggerChannelInterface::class) - ->add(LoggerChannelInterface::class, 'error', NULL, 'error') - ->add(MetastoreService::class, 'get', new RootedJsonData($json)) - ->add(AlterTableQueryFactoryInterface::class, 'setConnectionTimeout', AlterTableQueryFactoryInterface::class) - ->add(AlterTableQueryFactoryInterface::class, 'getQuery', AlterTableQueryInterface::class) - ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'resource_id') - ->add(PublicStream::class, 'getExternalUrl', self::HOST) - ->add(StreamWrapperManager::class, 'getViaUri', PublicStream::class) - ->add(ResourceMapper::class, 'get', Resource::class) - ->add(Resource::class, 'getVersion', $resource_version); - } - -} diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/PostImportResourceProcessorTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/PostImportResourceProcessorTest.php new file mode 100644 index 0000000000..d3be72c3c9 --- /dev/null +++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/PostImportResourceProcessorTest.php @@ -0,0 +1,184 @@ +add(ResourceProcessorInterface::class, 'process') + ->getMock(); + + $container_chain = $this->getContainerChain() + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) + ->add(ResourceMapper::class, 'get', $resource); + \Drupal::setContainer($container_chain->getMock()); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + $dictionaryEnforcer->processItem($resource); + + // Ensure resources were processed. + $notices = $container_chain->getStoredInput('notice'); + $this->assertEmpty($notices); + // Ensure no exceptions were thrown. + $errors = $container_chain->getStoredInput('error'); + $this->assertEmpty($errors); + } + + /** + * Test processItem() halts and logs a message if a resource no longer exists. + */ + public function testProcessItemResourceNoLongerExists() { + $resource = new Resource('test.csv', 'text/csv'); + + $resource_processor = (new Chain($this)) + ->add(ResourceProcessorInterface::class, 'process') + ->getMock(); + + $container_chain = $this->getContainerChain() + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) + ->add(ResourceMapper::class, 'get', NULL); + \Drupal::setContainer($container_chain->getMock()); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + $dictionaryEnforcer->processItem($resource); + + // Ensure notice was logged and resource processing was halted. + $notices = $container_chain->getStoredInput('notice'); + $this->assertEquals($notices[0], 'Cancelling resource processing; resource no longer exists.'); + // Ensure no exceptions were thrown. + $errors = $container_chain->getStoredInput('error'); + $this->assertEmpty($errors); + } + + /** + * Test processItem() halts and logs a message if a resource has changed. + */ + public function testProcessItemResourceChanged() { + $resource_a = new Resource('test.csv', 'text/csv'); + $resource_b = (new Resource('test2.csv', 'text/csv'))->createNewVersion(); + + $resource_processor = (new Chain($this)) + ->add(ResourceProcessorInterface::class, 'process') + ->getMock(); + + $container_chain = $this->getContainerChain() + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) + ->add(ResourceMapper::class, 'get', $resource_a); + \Drupal::setContainer($container_chain->getMock()); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + $dictionaryEnforcer->processItem($resource_b); + + // Ensure notice was logged and resource processing was halted. + $notices = $container_chain->getStoredInput('notice'); + $this->assertEquals($notices[0], 'Cancelling resource processing; resource has changed.'); + // Ensure no exceptions were thrown. + $errors = $container_chain->getStoredInput('error'); + $this->assertEmpty($errors); + } + + /** + * Test processItem() logs errors encountered in processors. + */ + public function testProcessItemProcessorError() { + $resource = new Resource('test.csv', 'text/csv'); + + $resource_processor = (new Chain($this)) + ->add(ResourceProcessorInterface::class, 'process', new \Exception('Test Error')) + ->getMock(); + + $container_chain = $this->getContainerChain() + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) + ->add(ResourceMapper::class, 'get', $resource); + \Drupal::setContainer($container_chain->getMock()); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + $dictionaryEnforcer->processItem($resource); + + // Ensure resources were processed. + $notices = $container_chain->getStoredInput('notice'); + $this->assertEmpty($notices); + // Ensure test error was caught. + $errors = $container_chain->getStoredInput('error'); + $this->assertEquals($errors[0], 'Test Error'); + } + + /** + * Get container chain. + */ + protected function getContainerChain() { + + $options = (new Options()) + ->add('dkan.datastore.data_dictionary.alter_table_query_factory.mysql', AlterTableQueryFactoryInterface::class) + ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscovery::class) + ->add('logger.factory', LoggerChannelFactoryInterface::class) + ->add('dkan.metastore.service', MetastoreService::class) + ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscoveryInterface::class) + ->add('stream_wrapper_manager', StreamWrapperManager::class) + ->add('dkan.metastore.resource_mapper', ResourceMapper::class) + ->add('dkan.datastore.service.resource_processor_collector', ResourceProcessorCollector::class) + ->index(0); + + $json = '{"identifier":"foo","title":"bar","data":{"fields":[]}}'; + + return (new Chain($this)) + ->add(Container::class, 'get', $options) + ->add(LoggerChannelFactoryInterface::class, 'get', LoggerChannelInterface::class) + ->add(LoggerChannelInterface::class, 'error', NULL, 'error') + ->add(LoggerChannelInterface::class, 'notice', NULL, 'notice') + ->add(MetastoreService::class, 'get', new RootedJsonData($json)) + ->add(AlterTableQueryFactoryInterface::class, 'setConnectionTimeout', AlterTableQueryFactoryInterface::class) + ->add(AlterTableQueryFactoryInterface::class, 'getQuery', AlterTableQueryInterface::class) + ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'resource_id') + ->add(PublicStream::class, 'getExternalUrl', self::HOST) + ->add(StreamWrapperManager::class, 'getViaUri', PublicStream::class) + ->add(ResourceMapper::class, 'get', Resource::class); + } + +} diff --git a/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php new file mode 100644 index 0000000000..2768dc5372 --- /dev/null +++ b/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php @@ -0,0 +1,176 @@ +add(AlterTableQueryFactoryInterface::class, 'getQuery', AlterTableQueryInterface::class) + ->add(AlterTableQueryInterface::class, 'applyDataTypes') + ->getMock(); + $metastore_service = (new Chain($this)) + ->add(MetastoreService::class, 'get', new RootedJsonData(json_encode(['data' => ['fields' => []]]))) + ->getMock(); + $dictionary_discovery_service = (new Chain($this)) + ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'dictionary-id') + ->getMock(); + $dictionary_enforcer = new DictionaryEnforcer($alter_table_query_factory, $metastore_service, $dictionary_discovery_service); + + $container_chain = $this->getContainerChain($resource->getVersion()) + ->add(AlterTableQueryInterface::class, 'applyDataTypes') + ->add(DataDictionaryDiscoveryInterface::class, 'getDataDictionaryMode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$dictionary_enforcer]); + \Drupal::setContainer($container_chain->getMock($resource->getVersion())); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + + $dictionaryEnforcer->processItem($resource); + + // Assert no exceptions are thrown. + $errors = $container_chain->getStoredInput('error'); + $this->assertEmpty($errors); + } + + /** + * Test exception thrown if no dictionary is found for resource. + */ + public function testNoDictionaryIdFoundForResourceException() { + $resource = new Resource('test.csv', 'text/csv'); + + $alter_table_query_factory = (new Chain($this)) + ->add(AlterTableQueryFactoryInterface::class, 'getQuery', AlterTableQueryInterface::class) + ->add(AlterTableQueryInterface::class, 'applyDataTypes') + ->getMock(); + $metastore_service = (new Chain($this)) + ->add(MetastoreService::class, 'get', new RootedJsonData(json_encode(['data' => ['fields' => []]]))) + ->getMock(); + $dictionary_discovery_service = (new Chain($this)) + ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', NULL) + ->getMock(); + $dictionary_enforcer = new DictionaryEnforcer($alter_table_query_factory, $metastore_service, $dictionary_discovery_service); + + $container_chain = $this->getContainerChain($resource->getVersion()) + ->add(AlterTableQueryInterface::class, 'applyDataTypes') + ->add(DataDictionaryDiscoveryInterface::class, 'getDataDictionaryMode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$dictionary_enforcer]); + \Drupal::setContainer($container_chain->getMock($resource->getVersion())); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + + $dictionaryEnforcer->processItem($resource); + + // Assert no exceptions are thrown. + $errors = $container_chain->getStoredInput('error'); + $this->assertEquals($errors[0], sprintf('No data-dictionary found for resource with id "%s" and version "%s".', $resource->getIdentifier(), $resource->getVersion())); + } + + /** + * Test exception thrown in applyDataTypes() is caught and logged. + */ + public function testProcessItemApplyDataTypesException() { + $resource = new Resource('test.csv', 'text/csv'); + + $alter_table_query_factory = (new Chain($this)) + ->add(AlterTableQueryFactoryInterface::class, 'getQuery', AlterTableQueryInterface::class) + ->add(AlterTableQueryInterface::class, 'applyDataTypes', new \Exception('Test Error')) + ->getMock(); + $metastore_service = (new Chain($this)) + ->add(MetastoreService::class, 'get', new RootedJsonData(json_encode(['data' => ['fields' => []]]))) + ->getMock(); + $dictionary_discovery_service = (new Chain($this)) + ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'data-dictionary') + ->getMock(); + $dictionary_enforcer = new DictionaryEnforcer($alter_table_query_factory, $metastore_service, $dictionary_discovery_service); + + $container_chain = $this->getContainerChain($resource->getVersion()) + ->add(AlterTableQueryInterface::class, 'applyDataTypes') + ->add(DataDictionaryDiscoveryInterface::class, 'getDataDictionaryMode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$dictionary_enforcer]); + \Drupal::setContainer($container_chain->getMock($resource->getVersion())); + + $dictionaryEnforcer = PostImportResourceProcessor::create( + $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + ); + + $dictionaryEnforcer->processItem($resource); + + // Assert no exceptions are thrown. + $errors = $container_chain->getStoredInput('error'); + $this->assertEquals($errors[0], 'Test Error'); + } + + /** + * Get container chain. + */ + protected function getContainerChain(int $resource_version) { + + $options = (new Options()) + ->add('dkan.datastore.data_dictionary.alter_table_query_factory.mysql', AlterTableQueryFactoryInterface::class) + ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscovery::class) + ->add('logger.factory', LoggerChannelFactoryInterface::class) + ->add('dkan.metastore.service', MetastoreService::class) + ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscoveryInterface::class) + ->add('stream_wrapper_manager', StreamWrapperManager::class) + ->add('dkan.metastore.resource_mapper', ResourceMapper::class) + ->add('dkan.datastore.service.resource_processor_collector', ResourceProcessorCollector::class) + ->index(0); + + $json = '{"identifier":"foo","title":"bar","data":{"fields":[]}}'; + + return (new Chain($this)) + ->add(Container::class, 'get', $options) + ->add(LoggerChannelFactoryInterface::class, 'get', LoggerChannelInterface::class) + ->add(LoggerChannelInterface::class, 'error', NULL, 'error') + ->add(MetastoreService::class, 'get', new RootedJsonData($json)) + ->add(AlterTableQueryFactoryInterface::class, 'setConnectionTimeout', AlterTableQueryFactoryInterface::class) + ->add(AlterTableQueryFactoryInterface::class, 'getQuery', AlterTableQueryInterface::class) + ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'resource_id') + ->add(PublicStream::class, 'getExternalUrl', self::HOST) + ->add(StreamWrapperManager::class, 'getViaUri', PublicStream::class) + ->add(ResourceMapper::class, 'get', Resource::class) + ->add(Resource::class, 'getVersion', $resource_version); + } + +} diff --git a/modules/metastore/config/schema/metastore.schema.yml b/modules/metastore/config/schema/metastore.schema.yml index 6e58bbbf54..f4e09f3186 100644 --- a/modules/metastore/config/schema/metastore.schema.yml +++ b/modules/metastore/config/schema/metastore.schema.yml @@ -2,10 +2,10 @@ metastore.settings: type: config_object label: 'Metastore Settings' mapping: - dictionary_mode: + data_dictionary_mode: type: string label: 'Dictionary Mode' - sitewide_dictionary_id: + data_dictionary_sitewide: type: string label: 'Sitewide Dictionary ID' property_list: diff --git a/modules/metastore/src/Form/DataDictionarySettingsForm.php b/modules/metastore/src/Form/DataDictionarySettingsForm.php index 0eb08f4e23..3913e61276 100644 --- a/modules/metastore/src/Form/DataDictionarySettingsForm.php +++ b/modules/metastore/src/Form/DataDictionarySettingsForm.php @@ -47,7 +47,7 @@ public function buildForm(array $form, FormStateInterface $form_state) { DataDictionaryDiscoveryInterface::MODE_NONE => $this->t('Disabled'), DataDictionaryDiscoveryInterface::MODE_SITEWIDE => $this->t('Sitewide'), ], - '#default_value' => $config->get('dictionary_mode'), + '#default_value' => $config->get('data_dictionary_mode'), '#attributes' => [ 'name' => 'dictionary_mode', ], @@ -61,7 +61,7 @@ public function buildForm(array $form, FormStateInterface $form_state) { ':input[name="dictionary_mode"]' => ['value' => DataDictionaryDiscoveryInterface::MODE_SITEWIDE], ], ], - '#default_value' => $config->get('sitewide_dictionary_id'), + '#default_value' => $config->get('data_dictionary_sitewide'), ]; return parent::buildForm($form, $form_state); @@ -74,8 +74,8 @@ public function submitForm(array &$form, FormStateInterface $form_state) { // Retrieve the configuration. $this->config(static::SETTINGS) // Set the submitted configuration setting. - ->set('dictionary_mode', $form_state->getValue('dictionary_mode')) - ->set('sitewide_dictionary_id', $form_state->getValue('sitewide_dictionary_id')) + ->set('data_dictionary_mode', $form_state->getValue('dictionary_mode')) + ->set('data_dictionary_sitewide', $form_state->getValue('sitewide_dictionary_id')) ->save(); parent::submitForm($form, $form_state); diff --git a/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php b/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php index a624fe06a6..292eab0159 100644 --- a/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php +++ b/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php @@ -30,12 +30,12 @@ public function provideFormData(): array { 'dictionary_mode' => [ '#value' => DataDictionaryDiscoveryInterface::MODE_SITEWIDE, '#config_name' => DataDictionarySettingsForm::SETTINGS, - '#config_key' => 'dictionary_mode', + '#config_key' => 'data_dictionary_mode', ], 'sitewide_dictionary_id' => [ '#value' => $this->randomString(), '#config_name' => DataDictionarySettingsForm::SETTINGS, - '#config_key' => 'sitewide_dictionary_id', + '#config_key' => 'data_dictionary_sitewide', ], ], ], @@ -44,7 +44,7 @@ public function provideFormData(): array { 'dictionary_mode' => [ '#value' => DataDictionaryDiscoveryInterface::MODE_NONE, '#config_name' => DataDictionarySettingsForm::SETTINGS, - '#config_key' => 'dictionary_mode', + '#config_key' => 'data_dictionary_mode', ], ], ],