From be785f08a9a5b752ceeab3a7238689b1445cb01d Mon Sep 17 00:00:00 2001 From: thierrydallacroce Date: Fri, 7 Jun 2019 08:46:40 -0700 Subject: [PATCH] References and endpoints for dataset properties and dynamic API routing (#129) --- cypress/integration/home.spec.js | 1 + modules/custom/dkan_api/dkan_api.routing.yml | 60 +- modules/custom/dkan_api/dkan_api.services.yml | 7 - .../custom/dkan_api/src/Controller/Api.php | 69 +- .../dkan_api/src/Controller/Dataset.php | 10 + .../dkan_api/src/Routing/RouteProvider.php | 93 ++ .../src/Storage/DrupalNodeDataset.php | 70 +- .../src/Storage/ThemeValueReferencer.php | 241 ----- .../Unit/Storage/DrupalNodeDatasetTest.php | 10 +- .../src/Unit/Storage/OrganizationTest.php | 2 +- .../Unit/Storage/ThemeValueReferencerTest.php | 518 ----------- .../dkan_common/dkan_common.links.menu.yml | 5 + .../dkan_common/dkan_common.routing.yml | 6 + .../config/install/dkan_data.settings.yml | 4 + .../custom/dkan_data/dkan_data.links.menu.yml | 5 + modules/custom/dkan_data/dkan_data.module | 44 +- .../custom/dkan_data/dkan_data.routing.yml | 8 + .../custom/dkan_data/dkan_data.services.yml | 7 + .../src/Form/DkanDataSettingsForm.php | 60 ++ .../QueueWorker/OrphanReferenceProcessor.php} | 50 +- .../custom/dkan_data/src/ValueReferencer.php | 405 +++++++++ ...est.php => ConfigurationOverriderTest.php} | 0 .../tests/src/Unit/ValueReferencerTest.php | 827 ++++++++++++++++++ modules/custom/dkan_harvest/src/Reverter.php | 1 + .../src/Controller/ApiController.php | 3 + .../src/Unit/Controller/ApiControllerTest.php | 1 + 26 files changed, 1596 insertions(+), 911 deletions(-) create mode 100644 modules/custom/dkan_api/src/Routing/RouteProvider.php delete mode 100644 modules/custom/dkan_api/src/Storage/ThemeValueReferencer.php delete mode 100644 modules/custom/dkan_api/tests/src/Unit/Storage/ThemeValueReferencerTest.php create mode 100644 modules/custom/dkan_common/dkan_common.links.menu.yml create mode 100644 modules/custom/dkan_common/dkan_common.routing.yml create mode 100644 modules/custom/dkan_data/config/install/dkan_data.settings.yml create mode 100644 modules/custom/dkan_data/dkan_data.links.menu.yml create mode 100644 modules/custom/dkan_data/dkan_data.routing.yml create mode 100644 modules/custom/dkan_data/src/Form/DkanDataSettingsForm.php rename modules/custom/{dkan_api/src/Plugin/QueueWorker/OrphanThemeProcessor.php => dkan_data/src/Plugin/QueueWorker/OrphanReferenceProcessor.php} (56%) create mode 100644 modules/custom/dkan_data/src/ValueReferencer.php rename modules/custom/dkan_data/tests/src/Unit/{CofigurationOverriderTest.php => ConfigurationOverriderTest.php} (100%) create mode 100644 modules/custom/dkan_data/tests/src/Unit/ValueReferencerTest.php diff --git a/cypress/integration/home.spec.js b/cypress/integration/home.spec.js index b5e13e06b3..898bb7b520 100755 --- a/cypress/integration/home.spec.js +++ b/cypress/integration/home.spec.js @@ -12,6 +12,7 @@ context('Home', () => { 'Public Safety', 'Transportation' ] + for (var key in topics) { var value = topics[key] var index = parseInt(key) + 1 diff --git a/modules/custom/dkan_api/dkan_api.routing.yml b/modules/custom/dkan_api/dkan_api.routing.yml index 8228734bf0..9e8e4eaf31 100644 --- a/modules/custom/dkan_api/dkan_api.routing.yml +++ b/modules/custom/dkan_api/dkan_api.routing.yml @@ -1,57 +1,3 @@ -dkan_api.dataset_get_all: - path: '/api/v1/dataset' - methods: [GET] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Dataset::getAll'} - requirements: - _access: 'TRUE' -dkan_api.dataset_get: - path: '/api/v1/dataset/{uuid}' - methods: [GET] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Dataset::get'} - requirements: - _access: 'TRUE' -dkan_api.dataset_post: - path: '/api/v1/dataset' - methods: [POST] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Dataset::post'} - options: - _auth: [ 'basic_auth' ] - requirements: - _permission: 'post put delete datasets through the api' -dkan_api.dataset_put: - path: '/api/v1/dataset/{uuid}' - methods: [PUT] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Dataset::put'} - options: - _auth: [ 'basic_auth' ] - requirements: - _permission: 'post put delete datasets through the api' -dkan_api.dataset_patch: - path: '/api/v1/dataset/{uuid}' - methods: [PATCH] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Dataset::patch'} - options: - _auth: [ 'basic_auth' ] - requirements: - _permission: 'post put delete datasets through the api' -dkan_api.dataset_delete: - path: '/api/v1/dataset/{uuid}' - methods: [DELETE] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Dataset::delete'} - options: - _auth: [ 'basic_auth' ] - requirements: - _permission: 'post put delete datasets through the api' -dkan_api.organization_get: - path: '/api/v1/organization' - methods: [GET] - defaults: - { _controller: '\Drupal\dkan_api\Controller\Organization::getAll'} - requirements: - _access: 'TRUE' +# Dynamic routes based on dkan_json_property_api's property_list config. +route_callbacks: + - '\Drupal\dkan_api\Routing\RouteProvider::routes' diff --git a/modules/custom/dkan_api/dkan_api.services.yml b/modules/custom/dkan_api/dkan_api.services.yml index 49ec3c5927..24b0617ec0 100644 --- a/modules/custom/dkan_api/dkan_api.services.yml +++ b/modules/custom/dkan_api/dkan_api.services.yml @@ -9,13 +9,6 @@ services: class: \Drupal\dkan_api\Storage\DrupalNodeDataset arguments: - '@entity_type.manager' - - '@dkan_api.storage.theme_value_referencer' dkan_api.storage.organization: class: \Drupal\dkan_api\Storage\Organization arguments: ['@dkan_api.storage.drupal_node_dataset'] - dkan_api.storage.theme_value_referencer: - class: \Drupal\dkan_api\Storage\ThemeValueReferencer - arguments: - - '@entity_type.manager' - - '@uuid' - - '@queue' diff --git a/modules/custom/dkan_api/src/Controller/Api.php b/modules/custom/dkan_api/src/Controller/Api.php index 18880cd924..384661eaa5 100644 --- a/modules/custom/dkan_api/src/Controller/Api.php +++ b/modules/custom/dkan_api/src/Controller/Api.php @@ -6,10 +6,19 @@ use Symfony\Component\DependencyInjection\ContainerInterface; /** + * Class Api. * + * @package Drupal\dkan_api\Controller */ abstract class Api extends ControllerBase { + /** + * Represents the data type passed via the HTTP request url schema_id slug. + * + * @var string + */ + protected $schemaId; + /** * Drupal service container. * @@ -25,7 +34,7 @@ abstract class Api extends ControllerBase { protected $dkanFactory; /** - * + * Api constructor. */ public function __construct(ContainerInterface $container) { $this->container = $container; @@ -33,21 +42,26 @@ public function __construct(ContainerInterface $container) { } /** - * + * Gets the json schema object. */ abstract protected function getJsonSchema(); /** - * + * Gets the storage object. */ abstract protected function getStorage(); /** * Get all. * + * @param string $schema_id + * The {schema_id} slug from the HTTP request. + * * @return \Symfony\Component\HttpFoundation\JsonResponse + * The json response. */ - public function getAll() { + public function getAll($schema_id = 'dataset') { + $this->schemaId = $schema_id; $datasets = $this->getEngine() ->get(); @@ -73,10 +87,15 @@ function ($json_string) { * * @param string $uuid * Identifier. + * @param string $schema_id + * The {schema_id} slug from the HTTP request. * - * @return \Symfony\Component\HttpFoundation\JsonResponse Json response. + * @return \Symfony\Component\HttpFoundation\JsonResponse + * The json response. */ - public function get($uuid) { + public function get($uuid, $schema_id = 'dataset') { + $this->schemaId = $schema_id; + try { $data = $this->getEngine() @@ -98,9 +117,15 @@ public function get($uuid) { /** * Implements POST method. * - * @return \Symfony\Component\HttpFoundation\JsonResponse Json response + * @param string $schema_id + * The {schema_id} slug from the HTTP request. + * + * @return \Symfony\Component\HttpFoundation\JsonResponse + * The json response. */ - public function post() { + public function post($schema_id = 'dataset') { + $this->schemaId = $schema_id; + /* @var $engine \Sae\Sae */ $engine = $this->getEngine(); @@ -142,10 +167,15 @@ public function post() { * * @param string $uuid * Identifier. + * @param string $schema_id + * The {schema_id} slug from the HTTP request. * - * @return \Symfony\Component\HttpFoundation\JsonResponse Json response + * @return \Symfony\Component\HttpFoundation\JsonResponse + * The json response. */ - public function put($uuid) { + public function put($uuid, $schema_id = 'dataset') { + $this->schemaId = $schema_id; + /* @var $engine \Sae\Sae */ $engine = $this->getEngine(); @@ -184,10 +214,15 @@ public function put($uuid) { * * @param string $uuid * Identifier. + * @param string $schema_id + * The {schema_id} slug from the HTTP request. * - * @return \Symfony\Component\HttpFoundation\JsonResponse Json response + * @return \Symfony\Component\HttpFoundation\JsonResponse + * The json response. */ - public function patch($uuid) { + public function patch($uuid, $schema_id = 'dataset') { + $this->schemaId = $schema_id; + /* @var $engine \Sae\Sae */ $engine = $this->getEngine(); @@ -229,10 +264,13 @@ public function patch($uuid) { * * @param string $uuid * Identifier. + * @param string $schema_id + * The {schema_id} slug from the HTTP request. * - * @return \Symfony\Component\HttpFoundation\JsonResponse Json response + * @return \Symfony\Component\HttpFoundation\JsonResponse + * The json response. */ - public function delete($uuid) { + public function delete($uuid, $schema_id = 'dataset') { /* @var $engine \Sae\Sae */ $engine = $this->getEngine(); @@ -242,9 +280,10 @@ public function delete($uuid) { } /** - * Get isntance of. + * Get SAE instance. * * @return \Sae\Sae + * Service Api Engine */ public function getEngine() { diff --git a/modules/custom/dkan_api/src/Controller/Dataset.php b/modules/custom/dkan_api/src/Controller/Dataset.php index 69179b1aa9..2af572f19e 100644 --- a/modules/custom/dkan_api/src/Controller/Dataset.php +++ b/modules/custom/dkan_api/src/Controller/Dataset.php @@ -31,6 +31,7 @@ public function __construct(ContainerInterface $container) { * @return \Drupal\dkan_api\Storage\DrupalNodeDataset Dataset */ protected function getStorage() { + $this->nodeDataset->setSchema('dataset'); return $this->nodeDataset; } @@ -38,9 +39,18 @@ protected function getStorage() { * Get Json Schema. * * @return string + * Json schema. */ protected function getJsonSchema() { + // @Todo: mechanism to validate against additional schemas. For now, + // validate against the empty object, as it accepts any valid json. + if (isset($this->schemaId) && $this->schemaId != 'dataset') { + // @codeCoverageIgnoreStart + return '{ }'; + // @codeCoverageIgnoreEnd + } + /** @var \Drupal\dkan_schema\SchemaRetriever $retriever */ $retriever = $this->container ->get('dkan_schema.schema_retriever'); diff --git a/modules/custom/dkan_api/src/Routing/RouteProvider.php b/modules/custom/dkan_api/src/Routing/RouteProvider.php new file mode 100644 index 0000000000..f13436ea94 --- /dev/null +++ b/modules/custom/dkan_api/src/Routing/RouteProvider.php @@ -0,0 +1,93 @@ +get('property_list'); + if ($list) { + // Trim and split list on newlines whether Windows, MacOS or Linux. + return preg_split( + '/\s*\r\n\s*|\s*\r\s*|\s*\n\s*/', + trim($list), + -1, + PREG_SPLIT_NO_EMPTY + ); + } + else { + return []; + } + } + + /** + * {@inheritdoc} + */ + public function routes() { + $routes = new RouteCollection(); + $public_routes = new RouteCollection(); + $authenticated_routes = new RouteCollection(); + $schemas = array_merge(['dataset'], $this->getPropertyList()); + + foreach ($schemas as $schema) { + // GET collection. + $get_all = $this->routeHelper($schema, "/api/v1/$schema", 'GET', 'getAll'); + $public_routes->add("dkan_api.{$schema}.get_all", $get_all); + // GET individual. + $get = $this->routeHelper($schema, "/api/v1/$schema/{uuid}", 'GET', 'get'); + $public_routes->add("dkan_api.{$schema}.get", $get); + // POST. + $post = $this->routeHelper($schema, "/api/v1/$schema", 'POST', 'post'); + $authenticated_routes->add("dkan_api.{$schema}.post", $post); + // PUT. + $put = $this->routeHelper($schema, "/api/v1/$schema/{uuid}", 'PUT', 'put'); + $authenticated_routes->add("dkan_api.{$schema}.put", $put); + // PATCH. + $patch = $this->routeHelper($schema, "/api/v1/$schema/{uuid}", 'PATCH', 'patch'); + $authenticated_routes->add("dkan_api.{$schema}.patch", $patch); + // DELETE. + $delete = $this->routeHelper($schema, "/api/v1/$schema/{uuid}", 'DELETE', 'delete'); + $authenticated_routes->add("dkan_api.{$schema}.delete", $delete); + } + + $public_routes->addRequirements(['_access' => 'TRUE']); + $authenticated_routes->addRequirements(['_permission' => 'post put delete datasets through the api']); + $authenticated_routes->addOptions(['_auth' => ['basic_auth']]); + $routes->addCollection($public_routes); + $routes->addCollection($authenticated_routes); + + return $routes; + } + + /** + * @param string $path + * @param string $datasetMethod + * @param string $httpVerb + * @return Route + */ + protected function routeHelper(string $schema, string $path, string $httpVerb, string $datasetMethod) : Route { + $route = new Route( + $path, + [ + '_controller' => '\Drupal\dkan_api\Controller\Dataset::' . $datasetMethod, + 'schema_id' => $schema, + ] + ); + $route->setMethods([$httpVerb]); + return $route; + } + +} diff --git a/modules/custom/dkan_api/src/Storage/DrupalNodeDataset.php b/modules/custom/dkan_api/src/Storage/DrupalNodeDataset.php index 8d490c9db0..58931d4413 100644 --- a/modules/custom/dkan_api/src/Storage/DrupalNodeDataset.php +++ b/modules/custom/dkan_api/src/Storage/DrupalNodeDataset.php @@ -21,26 +21,30 @@ class DrupalNodeDataset implements Storage { protected $entityTypeManager; /** - * Theme Value Referencer. + * Represents the data type passed via the HTTP request url schema_id slug. * - * @var Drupal\dkan_api\Storage\ThemeValueReferencer + * @var string */ - protected $themeValueReferencer; + protected $schemaId; /** * Constructor. * * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entityTypeManager * Injected entity type manager. - * @param \Drupal\dkan_api\Storage\ThemeValueReferencer $themeValueReferencer - * Injected theme value referencer. */ - public function __construct( - EntityTypeManagerInterface $entityTypeManager, - ThemeValueReferencer $themeValueReferencer - ) { + public function __construct(EntityTypeManagerInterface $entityTypeManager) { $this->entityTypeManager = $entityTypeManager; - $this->themeValueReferencer = $themeValueReferencer; + } + + /** + * Sets the data type. + * + * @param $schema_id string + * The HTTP request's schema or data type. + */ + public function setSchema($schema_id) { + $this->schemaId = $schema_id; } /** @@ -67,8 +71,12 @@ protected function getType() { */ public function retrieve(string $id): ?string { + if (!isset($this->schemaId)) { + throw new \Exception("DrupalNodeDataset schemaId not set in retrieve()."); + } + if (FALSE !== ($node = $this->getNodeByUuid($id))) { - return $this->themeDereferenced($node->field_json_metadata->value); + return $node->field_json_metadata->value; } throw new \Exception("No data with the identifier {$id} was found."); @@ -79,17 +87,21 @@ public function retrieve(string $id): ?string { */ public function retrieveAll(): array { + if (!isset($this->schemaId)) { + throw new \Exception("DrupalNodeDataset schemaId not set in retrieveAll()."); + } + $nodeStorage = $this->getNodeStorage(); $node_ids = $nodeStorage->getQuery() ->condition('type', $this->getType()) - ->condition('field_data_type', 'dataset') + ->condition('field_data_type', $this->schemaId) ->execute(); $all = []; foreach ($node_ids as $nid) { $node = $nodeStorage->load($nid); - $all[] = $this->themeDereferenced($node->field_json_metadata->value); + $all[] = $node->field_json_metadata->value; } return $all; } @@ -100,10 +112,6 @@ public function retrieveAll(): array { public function remove(string $id) { if (FALSE !== ($node = $this->getNodeByUuid($id))) { - // Check for orphan theme references. - $this->themeValueReferencer->processDeletedThemes( - $node->field_json_metadata->value - ); return $node->delete(); } } @@ -113,12 +121,12 @@ public function remove(string $id) { */ public function store(string $data, string $id = NULL): string { - $data = json_decode($data); - - if (isset($data->theme)) { - $data->theme = $this->themeValueReferencer->reference($data); + if (!isset($this->schemaId)) { + $this->schemaId = 'dataset'; } + $data = json_decode($data); + if (!$id && isset($data->identifier)) { $id = $data->identifier; } @@ -130,13 +138,8 @@ public function store(string $data, string $id = NULL): string { /* @var $node \Drupal\node\NodeInterface */ // update existing node if ($node) { - $node->field_data_type = "dataset"; + $node->field_data_type = $this->schemaId; $new_data = json_encode($data); - // Check for orphan theme references. - $this->themeValueReferencer->processDeletedThemes( - $node->field_json_metadata->value, - $new_data - ); $node->field_json_metadata = $new_data; $node->save(); return $node->uuid(); @@ -149,7 +152,7 @@ public function store(string $data, string $id = NULL): string { 'title' => $title, 'type' => 'data', 'uuid' => $id, - 'field_data_type' => 'dataset', + 'field_data_type' => $this->schemaId, 'field_json_metadata' => json_encode($data), ]); $node->save(); @@ -212,15 +215,4 @@ protected function getNodeByUuid($uuid) { return current($nodes); } - /** - * Helper function. - */ - protected function themeDereferenced($json) { - $data = json_decode($json); - if (isset($data->theme)) { - $data->theme = $this->themeValueReferencer->dereference($data); - } - return json_encode($data); - } - } diff --git a/modules/custom/dkan_api/src/Storage/ThemeValueReferencer.php b/modules/custom/dkan_api/src/Storage/ThemeValueReferencer.php deleted file mode 100644 index 54a04ff8a9..0000000000 --- a/modules/custom/dkan_api/src/Storage/ThemeValueReferencer.php +++ /dev/null @@ -1,241 +0,0 @@ -entityTypeManager = $entityTypeManager; - $this->uuidService = $uuidService; - $this->queueService = $queueService; - } - - /** - * Returns the uuid references for all themes values. - * - * @param \stdClass $data - * The object from the json data string. - * - * @return mixed - * An array of uuid, or NULL. - */ - public function reference(stdClass $data) { - if (!isset($data->theme) || !is_array($data->theme)) { - return NULL; - } - $themes = []; - foreach ($data->theme as $theme) { - $uuid = $this->referenceSingle($theme); - if (!$uuid) { - $uuid = $this->createThemeReference($theme); - } - // Return the existing or generated uuid, if not keep the original value. - if ($uuid) { - $themes[] = $uuid; - } - else { - $themes[] = $theme; - } - } - return $themes; - } - - /** - * Returns a single uuid reference for a particular theme value. - * - * If a corresponding existing uuid is not found, a theme data item is saved - * and its uuid returned. - * - * @param string $theme - * Human-readable theme value. - * - * @return mixed - * string containing uuid, or NULL. - */ - protected function referenceSingle(string $theme) { - $nodes = $this->entityTypeManager - ->getStorage('node') - ->loadByProperties([ - 'field_data_type' => "theme", - 'title' => $theme, - ]); - - if ($node = reset($nodes)) { - return $node->uuid->value; - } - return NULL; - } - - /** - * Generate and save a json theme item. - * - * @param string $theme - * Human-readable theme value. - * - * @return string - * The new theme data item's uuid. - */ - protected function createThemeReference(string $theme) { - $today = date('Y-m-d'); - - // Create theme json. - $data = new stdClass(); - $data->title = $theme; - $data->identifier = $this->uuidService->generate(); - $data->created = $today; - $data->modified = $today; - - // Create new data node for this theme. - $node = $this->entityTypeManager - ->getStorage('node') - ->create([ - 'title' => $theme, - 'type' => 'data', - 'uuid' => $data->identifier, - 'field_data_type' => 'theme', - 'field_json_metadata' => json_encode($data), - ]); - $node->save(); - - return $node->uuid(); - } - - /** - * Returns the human-readable theme values from uuids. - * - * @param \stdClass $data - * The object from the json data string. - * - * @return mixed - * An array of theme values, or NULL. - */ - public function dereference(stdClass $data) { - if (!isset($data->theme) || !is_array($data->theme)) { - return NULL; - } - $themes = []; - foreach ($data->theme as $theme) { - $themes[] = $this->dereferenceSingle($theme); - } - - if (!empty($themes)) { - return $themes; - } - else { - return NULL; - } - } - - /** - * Returns the human-readable theme value from its uuid. - * - * @param string $str - * The string could either be a uuid or a human-readable theme value. - * - * @return string - * The theme value. - */ - protected function dereferenceSingle(string $str) { - $nodes = $this->entityTypeManager - ->getStorage('node') - ->loadByProperties([ - 'field_data_type' => "theme", - 'uuid' => $str, - ]); - if ($node = reset($nodes)) { - return $node->title->value; - } - return $str; - } - - /** - * Queue deleted themes for processing, as they may be orphans. - * - * @param string $old - * Json string of item being replaced. - * @param string $new - * Json string of item doing the replacing. - * - * @return int - * The number of items queued for processing. - */ - public function processDeletedThemes(string $old, string $new = "{}") { - $themes_removed = $this->themesRemoved($old, $new); - - $orphan_theme_queue = $this->queueService->get('orphan_theme_processor'); - foreach ($themes_removed as $theme_removed) { - // @Todo: Only add to the queue when uuid doesn't already exists in it. - $orphan_theme_queue->createItem($theme_removed); - } - } - - /** - * Returns an array of theme uuid(s) being removed as the data changes. - * - * @param string $old - * Json string of item being replaced. - * @param string $new - * Json string of item doing the replacing. - * - * @return array - * Array of theme uuid(s). - */ - public function themesRemoved(string $old, string $new = "{}"): array { - $old_data = json_decode($old); - if (!isset($old_data->theme)) { - // No theme to potentially delete nor check for orphan. - return []; - } - $old_themes = $old_data->theme; - - $new_data = json_decode($new); - if (!isset($new_data->theme)) { - $new_themes = []; - } - else { - $new_themes = $new_data->theme; - } - - return array_diff($old_themes, $new_themes); - } - -} diff --git a/modules/custom/dkan_api/tests/src/Unit/Storage/DrupalNodeDatasetTest.php b/modules/custom/dkan_api/tests/src/Unit/Storage/DrupalNodeDatasetTest.php index 9a5e7ad7b5..df04b7c988 100644 --- a/modules/custom/dkan_api/tests/src/Unit/Storage/DrupalNodeDatasetTest.php +++ b/modules/custom/dkan_api/tests/src/Unit/Storage/DrupalNodeDatasetTest.php @@ -6,7 +6,6 @@ use Drupal\dkan_common\Tests\DkanTestBase; use Drupal\dkan_api\Storage\DrupalNodeDataset; use Drupal\node\NodeStorageInterface; -use Drupal\dkan_api\Storage\ThemeValueReferencer; use Drupal\dkan_datastore\Manager\DatastoreManagerBuilderHelper; use Drupal\dkan_datastore\Manager\DeferredImportQueuer; use Dkan\Datastore\Resource; @@ -26,22 +25,17 @@ class DrupalNodeDatasetTest extends DkanTestBase { */ public function testConstruct() { $mockEntityTypeManager = $this->createMock(EntityTypeManagerInterface::class); - $mockThemeValueReferencer = $this->createMock(ThemeValueReferencer::class); $mock = $this->getMockBuilder(DrupalNodeDataset::class) ->disableOriginalConstructor() ->getMock(); // Assert. - $mock->__construct($mockEntityTypeManager, $mockThemeValueReferencer); + $mock->__construct($mockEntityTypeManager); $this->assertSame( $mockEntityTypeManager, $this->readAttribute($mock, 'entityTypeManager') ); - $this->assertSame( - $mockThemeValueReferencer, - $this->readAttribute($mock, 'themeValueReferencer') - ); } /** @@ -205,7 +199,7 @@ public function testEnqueueDeferredImportOnException() { */ public function testRemainingMethods() { - $this->markTestIncomplete('Review of other methods in ' . DrupalNodeDataset::class . ' pending reivew of refactor.'); + $this->markTestIncomplete('Review of other methods in ' . DrupalNodeDataset::class . ' pending review of refactor.'); } } diff --git a/modules/custom/dkan_api/tests/src/Unit/Storage/OrganizationTest.php b/modules/custom/dkan_api/tests/src/Unit/Storage/OrganizationTest.php index 8d5048d7e7..9e9d9a7b4d 100644 --- a/modules/custom/dkan_api/tests/src/Unit/Storage/OrganizationTest.php +++ b/modules/custom/dkan_api/tests/src/Unit/Storage/OrganizationTest.php @@ -39,7 +39,7 @@ public function testConstruct() { */ public function testRemainingMethods() { - $this->markTestIncomplete('Review of other methods in ' . DrupalNodeDataset::class . ' pending reivew of refactor.'); + $this->markTestIncomplete('Review of other methods in ' . DrupalNodeDataset::class . ' pending review of refactor.'); } } diff --git a/modules/custom/dkan_api/tests/src/Unit/Storage/ThemeValueReferencerTest.php b/modules/custom/dkan_api/tests/src/Unit/Storage/ThemeValueReferencerTest.php deleted file mode 100644 index f42b7736bc..0000000000 --- a/modules/custom/dkan_api/tests/src/Unit/Storage/ThemeValueReferencerTest.php +++ /dev/null @@ -1,518 +0,0 @@ -getMockBuilder(ThemeValueReferencer::class) - ->setMethods(NULL) - ->disableOriginalConstructor() - ->getMock(); - - $mockEntityTypeManager = $this->createMock(EntityTypeManagerInterface::class); - $mockUuidInterface = $this->createMock(UuidInterface::class); - $mockQueueFactory = $this->createMock(QueueFactory::class); - - // Assert. - $mock->__construct($mockEntityTypeManager, $mockUuidInterface, $mockQueueFactory); - - $this->assertSame($mockEntityTypeManager, $this->readAttribute($mock, 'entityTypeManager')); - $this->assertSame($mockUuidInterface, $this->readAttribute($mock, 'uuidService')); - $this->assertSame($mockQueueFactory, $this->readAttribute($mock, 'queueService')); - } - - /** - * Provides data for testing referenceSingle function. - */ - public function dataTestReferenceSingle() { - $mockNode = $this->createMock(NodeInterface::class); - $expected = uniqid('a-uuid'); - $mockNode->uuid = (object) ['value' => $expected]; - - return [ - ['foobar', [$mockNode], $expected], - ['barfoo', [], NULL], - ]; - } - - /** - * Tests the referenceSingle function. - * - * @dataProvider dataTestReferenceSingle - * - * @param string $theme - * Theme uuid. - * @param array $nodes - * Array of node objects with uuid->value properties. - * @param mixed $expected - * Expected result. - */ - public function testReferenceSingle(string $theme, array $nodes, $expected) { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->setMethods(NULL) - ->disableOriginalConstructor() - ->getMock(); - - $mockEntityTypeManager = $this->getMockBuilder(EntityTypeManagerInterface::class) - ->setMethods([ - 'getStorage', - ]) - ->getMockForAbstractClass(); - - $this->writeProtectedProperty($mock, 'entityTypeManager', $mockEntityTypeManager); - - $mockNodeStorage = $this->getMockBuilder(EntityStorageInterface::class) - ->setMethods([ - 'loadByProperties', - ]) - ->getMockForAbstractClass(); - - // Expect. - $mockEntityTypeManager->expects($this->once()) - ->method('getStorage') - ->with('node') - ->willReturn($mockNodeStorage); - - $mockNodeStorage->expects($this->once()) - ->method('loadByProperties') - ->with([ - 'field_data_type' => "theme", - 'title' => $theme, - ]) - ->willReturn($nodes); - - // Assert. - $actual = $this->invokeProtectedMethod($mock, 'referenceSingle', $theme); - - $this->assertEquals($expected, $actual); - } - - /** - * Provides a list of old and new json strings to test themes being removed. - */ - public function dataTestThemesRemoved() { - return [ - ['{}', '{}', []], - ['{"theme":["Theme One"]}', '{"key":"value"}', ['Theme One']], - ['{"theme":["Theme One", "Theme Two"]}', '{"theme":["Theme One"]}', [1 => 'Theme Two']], - ]; - } - - /** - * Tests the themesRemoved function. - * - * @dataProvider dataTestThemesRemoved - * - * @param string $old - * Existing json string. - * @param string $new - * Incoming json string. - * @param array $expected - * Expected array containing themes removed between old and new. - */ - public function testThemesRemoved($old, $new, array $expected) { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(NULL) - ->getMock(); - - // Assert. - $this->assertEquals($expected, $mock->themesRemoved($old, $new)); - } - - /** - * Provides test data for function dereferenceSingle. - */ - public function dataTestDereferenceSingle() { - $mockNode = $this->createMock(NodeInterface::class); - $expected = uniqid('a-theme'); - $mockNode->title = (object) ['value' => $expected]; - - return [ - ['foobar', [$mockNode], $expected], - ['a-uuid-that-does-not-exist', [], 'a-uuid-that-does-not-exist'], - ]; - } - - /** - * Tests the dereferenceSingle function. - * - * @dataProvider dataTestDereferenceSingle - * - * @param string $str - * The human-readable theme value or a theme uuid. - * @param array $nodes - * An array of node objects with title->value property. - * @param mixed $expected - * Expected result from dereferenceSingle when passed test data. - */ - public function testDereferenceSingle($str, array $nodes, $expected) { - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->setMethods(NULL) - ->disableOriginalConstructor() - ->getMock(); - - $mockEntityTypeManager = $this->getMockBuilder(EntityTypeManagerInterface::class) - ->setMethods([ - 'getStorage', - ]) - ->getMockForAbstractClass(); - - $this->writeProtectedProperty($mock, 'entityTypeManager', $mockEntityTypeManager); - - $mockNodeStorage = $this->getMockBuilder(EntityStorageInterface::class) - ->setMethods([ - 'loadByProperties', - ]) - ->getMockForAbstractClass(); - - // Expect. - $mockEntityTypeManager->expects($this->once()) - ->method('getStorage') - ->with('node') - ->willReturn($mockNodeStorage); - - $mockNodeStorage->expects($this->once()) - ->method('loadByProperties') - ->with([ - 'field_data_type' => "theme", - 'uuid' => $str, - ]) - ->willReturn($nodes); - - // Assert. - $actual = $this->invokeProtectedMethod($mock, 'dereferenceSingle', $str); - $this->assertEquals($expected, $actual); - } - - /** - * Provides data without or empty themes to test function dereference. - */ - public function dataTestDereferenceWithoutThemes() { - return [ - [ - (object) ["non-theme" => "some value"], - NULL, - ], - [ - (object) ["theme" => "a string"], - NULL, - ], - [ - (object) ["theme" => []], - NULL, - ], - ]; - } - - /** - * Tests function dereference without valid theme values. - * - * @dataProvider dataTestDereferenceWithoutThemes - * - * @param \stdClass $data - * Object created from json data. - * @param mixed $expected - * Expected result from dereference when passed test data. - */ - public function testDereferenceWithoutThemes(\stdClass $data, $expected) { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(NULL) - ->getMock(); - - // Assert. - $this->assertEquals($expected, $mock->dereference($data)); - } - - /** - * Tests function dereference with valid theme values. - */ - public function testDereferenceWithThemes() { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(['dereferenceSingle']) - ->getMock(); - - $data = (object) ["theme" => [uniqid('theme-with-uuid-')]]; - $expected = 'corresponding-uuid'; - - // Expect. - $mock->expects($this->once()) - ->method('dereferenceSingle') - ->with($data->theme[0]) - ->willReturn($expected); - - // Assert. - $this->assertEquals([$expected], $mock->dereference($data)); - } - - /** - * Provides data without or empty themes to test function reference. - */ - public function dataTestReferenceWithoutThemes() { - return [ - [ - (object) ["non-theme" => "some value"], - NULL, - ], - [ - (object) ["theme" => "a string"], - NULL, - ], - [ - (object) ["theme" => []], - [], - ], - ]; - } - - /** - * Tests function reference without valid themes. - * - * @dataProvider dataTestReferenceWithoutThemes - * - * @param \stdClass $data - * Object created from json data. - * @param mixed $expected - * Expected result from dereference when passed test data. - */ - public function testReferenceWithoutThemes(\stdClass $data, $expected) { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(NULL) - ->getMock(); - - // Assert. - $this->assertEquals($expected, $mock->reference($data)); - } - - /** - * Tests function reference with valid theme values. - */ - public function testReferenceWithExistingThemes() { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(['referenceSingle']) - ->getMock(); - - $data = (object) ["theme" => [uniqid('theme-with-uuid-')]]; - $expect = 'corresponding-uuid'; - - // Expect. - $mock->expects($this->once()) - ->method('referenceSingle') - ->with($data->theme[0]) - ->willReturn($expect); - - // Assert. - $this->assertEquals([$expect], $mock->reference($data)); - } - - /** - * Tests the creation of new theme references. - */ - public function testReferenceCreatingNewThemeReference() { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(['referenceSingle', 'createThemeReference']) - ->getMock(); - - $data = (object) ["theme" => [uniqid('theme-with-uuid-')]]; - $expect = 'corresponding-uuid'; - - // Expect. - $mock->expects($this->once()) - ->method('referenceSingle') - ->with($data->theme[0]) - ->willReturn(NULL); - $mock->expects($this->once()) - ->method('createThemeReference') - ->with($data->theme[0]) - ->willReturn($expect); - - // Assert. - $this->assertEquals([$expect], $mock->reference($data)); - } - - /** - * Tests the outcome of failing to create a new theme reference, resulting - * in the theme value remaining its human-readable string. - */ - public function testReferenceFailsAtCreatingNewThemeReference() { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(['referenceSingle', 'createThemeReference']) - ->getMock(); - - $human_readable_value = uniqid('human-readable-theme-value'); - $data = (object) ["theme" => [$human_readable_value]]; - - // Expect. - $mock->expects($this->once()) - ->method('referenceSingle') - ->with($data->theme[0]) - ->willReturn(NULL); - $mock->expects($this->once()) - ->method('createThemeReference') - ->with($data->theme[0]) - ->willReturn(FALSE); - - // Assert. - $this->assertEquals([$human_readable_value], $mock->reference($data)); - } - - /** - * Tests the processDeletedThemes function. - */ - public function testProcessDeletedThemes() { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods([ - 'themesRemoved', - ]) - ->getMock(); - $mockQueueFactory = $this->getMockBuilder(QueueFactory::class) - ->disableOriginalConstructor() - ->setMethods([ - 'get', - ]) - ->getMockForAbstractClass(); - $this->writeProtectedProperty($mock, 'queueService', $mockQueueFactory); - - $mockQueueInterface = $this->getMockBuilder(QueueInterface::class) - ->setMethods([ - 'createItem', - ]) - ->getMockForAbstractClass(); - - $old = '{"theme":["Theme One", "Theme Two", "Theme Three"]}'; - $new = '{"theme":["Theme Two"]}'; - $themes_removed = ["Theme One", "Theme Three"]; - - // Expect. - $mock->expects($this->once()) - ->method('themesRemoved') - ->with($old, $new) - ->willReturn($themes_removed); - $mockQueueFactory->expects($this->atLeastOnce()) - ->method('get') - ->with('orphan_theme_processor') - ->willReturn($mockQueueInterface); - $mockQueueInterface->expects($this->exactly(2)) - ->method('createItem') - ->withConsecutive([$themes_removed[0]], [$themes_removed[1]]) - ->willReturnOnConsecutiveCalls(TRUE, TRUE); - - $mock->processDeletedThemes($old, $new); - } - - /** - * Tests the createThemeReference function. - */ - public function testCreateThemeReference() { - // Setup. - $mock = $this->getMockBuilder(ThemeValueReferencer::class) - ->disableOriginalConstructor() - ->setMethods(NULL) - ->getMock(); - - $mockUuidInterface = $this->getMockBuilder(UuidInterface::class) - ->disableOriginalConstructor() - ->setMethods([ - 'generate', - ]) - ->getMockForAbstractClass(); - $this->writeProtectedProperty($mock, 'uuidService', $mockUuidInterface); - - $mockEntityTypeManager = $this->getMockBuilder(EntityTypeManagerInterface::class) - ->setMethods([ - 'getStorage', - ]) - ->getMockForAbstractClass(); - $this->writeProtectedProperty($mock, 'entityTypeManager', $mockEntityTypeManager); - - $mockNodeStorage = $this->getMockBuilder(EntityStorageInterface::class) - ->setMethods([ - 'create', - ]) - ->getMockForAbstractClass(); - - $mockEntityInterface = $this->getMockBuilder(EntityInterface::class) - ->setMethods([ - 'save', - 'uuid', - ]) - ->getMockForAbstractClass(); - - $theme = uniqid('some-theme-'); - $uuid = uniqid('some-uuid-'); - $today = date('Y-m-d'); - $data = new stdClass(); - $data->title = $theme; - $data->identifier = $uuid; - $data->created = $today; - $data->modified = $today; - - // Expect. - $mockUuidInterface->expects($this->once()) - ->method('generate') - ->willReturn($uuid); - $mockEntityTypeManager->expects($this->once()) - ->method('getStorage') - ->with('node') - ->willReturn($mockNodeStorage); - $mockNodeStorage->expects($this->once()) - ->method('create') - ->with([ - 'title' => $theme, - 'type' => 'data', - 'uuid' => $uuid, - 'field_data_type' => 'theme', - 'field_json_metadata' => json_encode($data), - ]) - ->willReturn($mockEntityInterface); - $mockEntityInterface->expects($this->once()) - ->method('save') - ->willReturn(1); - $mockEntityInterface->expects($this->once()) - ->method('uuid') - ->willReturn($uuid); - - // Assert. - $actual = $this->invokeProtectedMethod($mock, 'createThemeReference', $theme); - $this->assertEquals($uuid, $actual); - } - -} diff --git a/modules/custom/dkan_common/dkan_common.links.menu.yml b/modules/custom/dkan_common/dkan_common.links.menu.yml new file mode 100644 index 0000000000..abee4c50be --- /dev/null +++ b/modules/custom/dkan_common/dkan_common.links.menu.yml @@ -0,0 +1,5 @@ +dkan_common.admin_config_dkan: + route_name: dkan_common.admin_config_dkan + title: DKAN + parent: system.admin_config + description: Modify DKAN configuration. diff --git a/modules/custom/dkan_common/dkan_common.routing.yml b/modules/custom/dkan_common/dkan_common.routing.yml new file mode 100644 index 0000000000..bd8577cd1b --- /dev/null +++ b/modules/custom/dkan_common/dkan_common.routing.yml @@ -0,0 +1,6 @@ +dkan_common.admin_config_dkan: + path: '/admin/config/dkan' + defaults: + _controller: '\Drupal\system\Controller\SystemController::systemAdminMenuBlockPage' + requirements: + _permission: 'access administration pages' diff --git a/modules/custom/dkan_data/config/install/dkan_data.settings.yml b/modules/custom/dkan_data/config/install/dkan_data.settings.yml new file mode 100644 index 0000000000..230817fad9 --- /dev/null +++ b/modules/custom/dkan_data/config/install/dkan_data.settings.yml @@ -0,0 +1,4 @@ +property_list: | + theme + keyword + organization diff --git a/modules/custom/dkan_data/dkan_data.links.menu.yml b/modules/custom/dkan_data/dkan_data.links.menu.yml new file mode 100644 index 0000000000..22bcb7a89b --- /dev/null +++ b/modules/custom/dkan_data/dkan_data.links.menu.yml @@ -0,0 +1,5 @@ +dkan_data.api_settings_form: + title: Dataset properties + route_name: dkan_data.config_settings + description: Configure dataset properties for referencing and API endpoint. + parent: dkan_common.admin_config_dkan \ No newline at end of file diff --git a/modules/custom/dkan_data/dkan_data.module b/modules/custom/dkan_data/dkan_data.module index 813b730fb4..e4bf68d3bf 100644 --- a/modules/custom/dkan_data/dkan_data.module +++ b/modules/custom/dkan_data/dkan_data.module @@ -14,12 +14,14 @@ function dkan_data_node_load(array $entities) { foreach ($entities as $entity) { if ($entity->bundle() == "data" && $entity->field_data_type->value == "dataset") { - $metadata = json_decode($entity->get('field_json_metadata')->value); - if (isset($metadata->theme)) { - $referencer = Drupal::service("dkan_api.storage.theme_value_referencer"); - $metadata->theme = $referencer->dereference($metadata); - $entity->set('field_json_metadata', json_encode($metadata)); - } + // Temporarily save the raw json metadata, for later use. + $metadata_string = $entity->get('field_json_metadata')->value; + $entity->referenced_metadata = $metadata_string; + // Dereference dataset properties. + $metadata_obj = json_decode($metadata_string); + $referencer = Drupal::service("dkan_data.value_referencer"); + $metadata_obj = $referencer->dereference($metadata_obj); + $entity->set('field_json_metadata', json_encode($metadata_obj)); } } @@ -34,6 +36,10 @@ function dkan_data_entity_presave(EntityInterface $entity) { return; } + if ($entity->get('field_data_type')->value != 'dataset') { + return; + } + $entityType = $entity->getEntityTypeId(); $metadata = json_decode($entity->get('field_json_metadata')->value); @@ -55,12 +61,30 @@ function dkan_data_entity_presave(EntityInterface $entity) { $entity->set('uuid', $metadata->identifier); } - if (isset($metadata->theme)) { - $referencer = Drupal::service("dkan_api.storage.theme_value_referencer"); - $metadata->theme = $referencer->reference($metadata); + // Reference the dataset's values, and update our json metadata. + $referencer = Drupal::service("dkan_data.value_referencer"); + $metadata = $referencer->reference($metadata); + $entity->set('field_json_metadata', json_encode($metadata)); + + // Check for possible orphan property references when updating a dataset. + if (isset($entity->original)) { + $referencer->processReferencesInUpdatedDataset( + json_decode($entity->referenced_metadata), + $metadata + ); } +} - $entity->set('field_json_metadata', json_encode($metadata)); +/** + * Implements hook_ENTITY_TYPE_predelete(). + */ +function dkan_data_node_predelete(EntityInterface $entity) { + // Check for possible orphan property references when deleting a dataset. + if ($entity->bundle() == 'data' && $entity->get('field_data_type')->value == 'dataset') { + $referenced_dataset = json_decode($entity->referenced_metadata); + $referencer = Drupal::service("dkan_data.value_referencer"); + $referencer->processReferencesInDeletedDataset($referenced_dataset); + } } /** diff --git a/modules/custom/dkan_data/dkan_data.routing.yml b/modules/custom/dkan_data/dkan_data.routing.yml new file mode 100644 index 0000000000..5ccd7894ce --- /dev/null +++ b/modules/custom/dkan_data/dkan_data.routing.yml @@ -0,0 +1,8 @@ +dkan_data.config_settings: + path: '/admin/config/dkan/properties' + defaults: + _form: '\Drupal\dkan_data\Form\DkanDataSettingsForm' + requirements: + _permission: 'access administration pages' + options: + _admin_route: TRUE diff --git a/modules/custom/dkan_data/dkan_data.services.yml b/modules/custom/dkan_data/dkan_data.services.yml index bf10abb49a..1335c4bcfa 100644 --- a/modules/custom/dkan_data/dkan_data.services.yml +++ b/modules/custom/dkan_data/dkan_data.services.yml @@ -3,3 +3,10 @@ services: class: \Drupal\dkan_data\ConfigurationOverrider tags: - {name: config.factory.override, priority: 5} + dkan_data.value_referencer: + class: \Drupal\dkan_data\ValueReferencer + arguments: + - '@entity_type.manager' + - '@uuid' + - '@config.factory' + - '@queue' diff --git a/modules/custom/dkan_data/src/Form/DkanDataSettingsForm.php b/modules/custom/dkan_data/src/Form/DkanDataSettingsForm.php new file mode 100644 index 0000000000..a6d46e94fd --- /dev/null +++ b/modules/custom/dkan_data/src/Form/DkanDataSettingsForm.php @@ -0,0 +1,60 @@ +config('dkan_data.settings'); + $form['property_list'] = [ + '#type' => 'textarea', + '#title' => $this->t('List of dataset properties to be referenced'), + '#description' => $this->t('Separate properties by a new line.'), + '#default_value' => $config->get('property_list'), + ]; + return parent::buildForm($form, $form_state); + } + + /** + * {@inheritdoc} + */ + public function submitForm(array &$form, FormStateInterface $form_state) { + parent::submitForm($form, $form_state); + + $this->config('dkan_data.settings') + ->set('property_list', $form_state->getValue('property_list')) + ->save(); + + // Rebuild routes, without clearing all caches. + \Drupal::service("router.builder")->rebuild(); + } + +} diff --git a/modules/custom/dkan_api/src/Plugin/QueueWorker/OrphanThemeProcessor.php b/modules/custom/dkan_data/src/Plugin/QueueWorker/OrphanReferenceProcessor.php similarity index 56% rename from modules/custom/dkan_api/src/Plugin/QueueWorker/OrphanThemeProcessor.php rename to modules/custom/dkan_data/src/Plugin/QueueWorker/OrphanReferenceProcessor.php index ff3f1ffd57..c222e8a911 100644 --- a/modules/custom/dkan_api/src/Plugin/QueueWorker/OrphanThemeProcessor.php +++ b/modules/custom/dkan_data/src/Plugin/QueueWorker/OrphanReferenceProcessor.php @@ -2,7 +2,7 @@ declare(strict_types = 1); -namespace Drupal\dkan_api\Plugin\QueueWorker; +namespace Drupal\dkan_data\Plugin\QueueWorker; use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; @@ -10,15 +10,17 @@ use Symfony\Component\DependencyInjection\ContainerInterface; /** - * Verifies if a theme is orphaned, then deletes it. + * Verifies if a dataset property reference is orphaned, then deletes it. * * @QueueWorker( - * id = "orphan_theme_processor", - * title = @Translation("Task Worker: Verify then delete orphaned theme"), + * id = "orphan_reference_processor", + * title = @Translation("Task Worker: Check for orphaned property reference"), * cron = {"time" = 15} * ) + * + * @codeCoverageIgnore */ -class OrphanThemeProcessor extends QueueWorkerBase implements ContainerFactoryPluginInterface { +class OrphanReferenceProcessor extends QueueWorkerBase implements ContainerFactoryPluginInterface { /** * The entity type manager service. @@ -59,29 +61,47 @@ public static function create(ContainerInterface $container, array $configuratio /** * {@inheritdoc} */ - public function processItem($uuid) { + public function processItem($data) { + $property_id = $data[0]; + $uuid = $data[1]; + + // Search datasets using this uuid for this property id. $datasets = $this->entityTypeManager->getStorage('node') ->loadByProperties([ 'field_data_type' => 'dataset', ]); - foreach ($datasets as $dataset) { - $data = json_decode($dataset->field_json_metadata->value); - $themes = $data->theme ?? []; - if (in_array($uuid, $themes)) { + $data = json_decode($dataset->referenced_metadata); + $value = $data->{$property_id}; + // Check if uuid is found either directly or in an array. + $uuid_is_value = $uuid == $value; + $uuid_found_in_array = is_array($value) && in_array($uuid, $value); + if ($uuid_is_value || $uuid_found_in_array) { // Uuid found in use, abort. return; } } - // Theme uuid not found in any dataset, safe to delete. - $themes = $this->entityTypeManager->getStorage('node') + // Value reference uuid not found in any dataset, therefore safe to delete. + $this->deleteReference($property_id, $uuid); + } + + /** + * Deletes a reference. + * + * @param string $property_id + * The property id. + * @param string $uuid + * The uuid. + */ + protected function deleteReference(string $property_id, string $uuid) { + $references = $this->entityTypeManager->getStorage('node') ->loadByProperties([ - 'field_data_type' => 'theme', + 'field_data_type' => $property_id, 'uuid' => $uuid, ]); - if (FALSE !== ($theme = reset($themes))) { - $theme->delete(); + if (FALSE !== ($reference = reset($references))) { + $reference->delete(); } } diff --git a/modules/custom/dkan_data/src/ValueReferencer.php b/modules/custom/dkan_data/src/ValueReferencer.php new file mode 100644 index 0000000000..1f9d6bb2d1 --- /dev/null +++ b/modules/custom/dkan_data/src/ValueReferencer.php @@ -0,0 +1,405 @@ +entityTypeManager = $entityTypeManager; + $this->uuidService = $uuidService; + $this->configService = $configService; + $this->queueService = $queueService; + } + + /** + * Replaces some dataset property values with references. + * + * @param \stdClass $data + * Dataset json object. + * + * @return \stdClass + * Json object modified with references to some of its properties' values. + */ + public function reference(stdClass $data) { + // Cycle through the dataset properties we seek to reference. + foreach ($this->getPropertyList() as $property_id) { + if (isset($data->{$property_id})) { + $data->{$property_id} = $this->referenceProperty($property_id, $data->{$property_id}); + } + } + return $data; + } + + /** + * References a dataset property's value, general case. + * + * @param string $property_id + * The dataset property id. + * @param mixed $data + * Single value or array of values to be referenced. + * + * @return string|array + * Single reference, or an array of references. + */ + protected function referenceProperty(string $property_id, $data) { + if (is_array($data)) { + return $this->referenceMultiple($property_id, $data); + } + else { + // Object or string. + return $this->referenceSingle($property_id, $data); + } + } + + /** + * References a dataset property's value, array case. + * + * @param string $property_id + * The dataset property id. + * @param array $values + * The array of values to be referenced. + * + * @return array + * The array of uuid references. + */ + protected function referenceMultiple(string $property_id, array $values) : array { + $result = []; + foreach ($values as $value) { + $result[] = $this->referenceSingle($property_id, $value); + } + return $result; + } + + /** + * References a dataset property's value, string or object case. + * + * @param string $property_id + * The dataset property id. + * @param string|object $value + * The value to be referenced. + * + * @return string + * The Uuid reference, or unchanged value. + */ + protected function referenceSingle(string $property_id, $value) { + $uuid = $this->checkExistingReference($property_id, $value); + if (!$uuid) { + $uuid = $this->createPropertyReference($property_id, $value); + } + if ($uuid) { + return $uuid; + } + else { + // In the unlikely case we neither found an existing reference nor could + // create a new reference, return the unchanged value. + return $value; + } + } + + /** + * Checks for an existing value reference for that property id. + * + * @param string $property_id + * The dataset property id. + * @param $data + * The property's value used to find an existing reference. + * + * @return string|null + * The existing reference's uuid, or null if not found. + */ + protected function checkExistingReference(string $property_id, $data) { + $nodes = $this->entityTypeManager + ->getStorage('node') + ->loadByProperties([ + 'field_data_type' => $property_id, + 'title' => md5(json_encode($data)), + ]); + + if ($node = reset($nodes)) { + return $node->uuid->value; + } + return NULL; + } + + /** + * Creates a new value reference for that property id in a data node. + * + * @param string $property_id + * The dataset property id. + * @param mixed $value + * The property's value. + * + * @return string|null + * The new reference's uuid, or null. + */ + protected function createPropertyReference(string $property_id, $value) { + // Create json metadata for the reference. + $data = new stdClass(); + $data->identifier = $this->uuidService->generate(); + $data->data = $value; + + // Create node to store this reference. + $node = $this->entityTypeManager + ->getStorage('node') + ->create([ + 'title' => md5(json_encode($value)), + 'type' => 'data', + 'uuid' => $data->identifier, + 'field_data_type' => $property_id, + 'field_json_metadata' => json_encode($data), + ]); + $node->save(); + + return $node->uuid(); + } + + /** + * Replaces value references in a dataset with with their actual values. + * + * @param \stdClass $data + * The json metadata object. + * + * @return mixed + * Modified json metadata object. + */ + public function dereference(stdClass $data) { + // Cycle through the dataset properties we seek to dereference. + foreach ($this->getPropertyList() as $property_id) { + if (isset($data->{$property_id})) { + $data->{$property_id} = $this->dereferenceProperty($property_id, $data->{$property_id}); + } + } + return $data; + } + + /** + * Replaces a property reference with its actual value, general case. + * + * @param string $property_id + * The dataset property id. + * @param string|array $uuids + * A single reference uuid string, or an array reference uuids. + * + * @return string|array + * An array of dereferenced values, or a single one. + */ + protected function dereferenceProperty(string $property_id, $data) { + if (is_array($data)) { + return $this->dereferenceMultiple($property_id, $data); + } + else { + return $this->dereferenceSingle($property_id, $data); + } + } + + /** + * Replaces a property reference with its actual value, array case. + * + * @param string $property_id + * A dataset property id. + * @param array $uuids + * An array of reference uuids. + * + * @return array + * An array of dereferenced values. + */ + protected function dereferenceMultiple(string $property_id, array $uuids) : array { + $result = []; + foreach ($uuids as $uuid) { + $result[] = $this->dereferenceSingle($property_id, $uuid); + } + return $result; + } + + /** + * Replaces a property reference with its actual value, string or object case. + * + * @param string $property_id + * The dataset property id. + * @param string $str + * Either a uuid or an actual json value. + * + * @return string + * The data from this reference. + */ + protected function dereferenceSingle(string $property_id, string $uuid) { + $nodes = $this->entityTypeManager + ->getStorage('node') + ->loadByProperties([ + 'field_data_type' => $property_id, + 'uuid' => $uuid, + ]); + if ($node = reset($nodes)) { + if (isset($node->field_json_metadata->value)) { + $metadata = json_decode($node->field_json_metadata->value); + return $metadata->data; + } + } + // If str was not found, it's unlikely it was a uuid to begin with. It was + // most likely never referenced to begin with, so return unchanged. + return $uuid; + } + + /** + * Check for orphan references when a dataset is being deleted. + * + * @param \stdClass $data + * Dataset to be deleted. + */ + public function processReferencesInDeletedDataset(stdClass $data) { + // Cycle through the dataset properties we seek to reference. + foreach ($this->getPropertyList() as $property_id) { + if (isset($data->{$property_id})) { + $this->processReferencesInDeletedProperty($property_id, $data->{$property_id}); + } + } + } + + /** + * + */ + protected function processReferencesInDeletedProperty($property_id, $uuids) { + // Treat single uuid as an array of one uuid. + if (!is_array($uuids)) { + $uuids = [$uuids]; + } + foreach ($uuids as $uuid) { + $this->queueReferenceForRemoval($property_id, $uuid); + } + } + + /** + * @param $property_id + * @param $uuid + * + * @codeCoverageIgnore since no logic, single call to queue worker. + */ + protected function queueReferenceForRemoval($property_id, $uuid) { + $this->queueService->get('orphan_reference_processor') + ->createItem([ + $property_id, + $uuid, + ]); + } + + /** + * + */ + public function processReferencesInUpdatedDataset(stdClass $old_dataset, stdClass $new_dataset) { + // Cycle through the dataset properties being referenced, check for orphans. + foreach ($this->getPropertyList() as $property_id) { + if (!isset($old_dataset->{$property_id})) { + // The old dataset had no value for this property, thus no references + // could be deleted. Safe to skip checking for orphan reference. + continue; + } + if (!isset($new_dataset->{$property_id})) { + $new_dataset->{$property_id} = $this->emptyPropertyOfSameType($old_dataset->{$property_id}); + } + $this->processReferencesInUpdatedProperty($property_id, $old_dataset->{$property_id}, $new_dataset->{$property_id}); + } + } + + /** + * + */ + protected function processReferencesInUpdatedProperty($property_id, $old_value, $new_value) { + if (!is_array($old_value)) { + $old_value = [$old_value]; + $new_value = [$new_value]; + } + foreach (array_diff($old_value, $new_value) as $removed_reference) { + $this->queueReferenceForRemoval($property_id, $removed_reference); + } + } + + /** + * @param $data + * + * @return array|string + */ + protected function emptyPropertyOfSameType($data) { + if (is_array($data)) { + return []; + } + return ""; + } + + /** + * Get the list of dataset properties being referenced. + * + * @return array + * List of dataset properties. + * + * @Todo: consolidate with dkan_api RouteProvider's getPropertyList. + */ + protected function getPropertyList() : array { + $list = $this->configService->get('dkan_data.settings')->get('property_list'); + if ($list) { + // Trim and split list on newlines whether Windows, MacOS or Linux. + return preg_split( + '/\s*\r\n\s*|\s*\r\s*|\s*\n\s*/', + trim($list), + -1, + PREG_SPLIT_NO_EMPTY + ); + } + else { + return []; + } + } + +} diff --git a/modules/custom/dkan_data/tests/src/Unit/CofigurationOverriderTest.php b/modules/custom/dkan_data/tests/src/Unit/ConfigurationOverriderTest.php similarity index 100% rename from modules/custom/dkan_data/tests/src/Unit/CofigurationOverriderTest.php rename to modules/custom/dkan_data/tests/src/Unit/ConfigurationOverriderTest.php diff --git a/modules/custom/dkan_data/tests/src/Unit/ValueReferencerTest.php b/modules/custom/dkan_data/tests/src/Unit/ValueReferencerTest.php new file mode 100644 index 0000000000..cfd805686e --- /dev/null +++ b/modules/custom/dkan_data/tests/src/Unit/ValueReferencerTest.php @@ -0,0 +1,827 @@ +getMockBuilder(ValueReferencer::class) + ->setMethods(NULL) + ->disableOriginalConstructor() + ->getMock(); + + $mockEntityTypeManager = $this->createMock(EntityTypeManagerInterface::class); + $mockUuidInterface = $this->createMock(UuidInterface::class); + $mockConfigInterface = $this->createMock(ConfigFactoryInterface::class); + $mockQueueFactory = $this->createMock(QueueFactory::class); + + // Assert. + $mock->__construct($mockEntityTypeManager, $mockUuidInterface, $mockConfigInterface, $mockQueueFactory); + + $this->assertSame($mockEntityTypeManager, $this->readAttribute($mock, 'entityTypeManager')); + $this->assertSame($mockUuidInterface, $this->readAttribute($mock, 'uuidService')); + $this->assertSame($mockConfigInterface, $this->readAttribute($mock, 'configService')); + $this->assertSame($mockQueueFactory, $this->readAttribute($mock, 'queueService')); + } + + /** + * Provides data for testing checkExistingReference function. + */ + public function dataTestCheckExistingReference() { + $mockNode = $this->createMock(NodeInterface::class); + $expected = uniqid('a-uuid'); + $mockNode->uuid = (object) ['value' => $expected]; + + return [ + ['theme', 'Topic One', [$mockNode], $expected], + ['barfoo', '', [], NULL], + ]; + } + + /** + * Tests the checkExistingReference function. + * + * @param string $property_id + * The property name. + * @param mixed $data + * The json value of the property. + * @param array $nodes + * Array of node objects with uuid->value properties. + * @param mixed $expected + * Expected result. + * + * @dataProvider dataTestCheckExistingReference + */ + public function testCheckExistingReference($property_id, $data, array $nodes, $expected) { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(NULL) + ->disableOriginalConstructor() + ->getMock(); + + $mockEntityTypeManager = $this->getMockBuilder(EntityTypeManagerInterface::class) + ->setMethods([ + 'getStorage', + ]) + ->getMockForAbstractClass(); + + $this->writeProtectedProperty($mock, 'entityTypeManager', $mockEntityTypeManager); + + $mockNodeStorage = $this->getMockBuilder(EntityStorageInterface::class) + ->setMethods([ + 'loadByProperties', + ]) + ->getMockForAbstractClass(); + + // Expect. + $mockEntityTypeManager->expects($this->once()) + ->method('getStorage') + ->with('node') + ->willReturn($mockNodeStorage); + + $mockNodeStorage->expects($this->once()) + ->method('loadByProperties') + ->with([ + 'field_data_type' => $property_id, + 'title' => md5(json_encode($data)), + ]) + ->willReturn($nodes); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'checkExistingReference', $property_id, $data); + $this->assertEquals($expected, $actual); + } + + /** + * Tests the createPropertyReference function. + */ + public function testCreatePropertyReference() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(NULL) + ->getMock(); + + $mockUuidInterface = $this->getMockBuilder(UuidInterface::class) + ->disableOriginalConstructor() + ->setMethods([ + 'generate', + ]) + ->getMockForAbstractClass(); + $this->writeProtectedProperty($mock, 'uuidService', $mockUuidInterface); + + $mockEntityTypeManager = $this->getMockBuilder(EntityTypeManagerInterface::class) + ->setMethods([ + 'getStorage', + ]) + ->getMockForAbstractClass(); + $this->writeProtectedProperty($mock, 'entityTypeManager', $mockEntityTypeManager); + + $mockNodeStorage = $this->getMockBuilder(EntityStorageInterface::class) + ->setMethods([ + 'create', + ]) + ->getMockForAbstractClass(); + + $mockEntityInterface = $this->getMockBuilder(EntityInterface::class) + ->setMethods([ + 'save', + 'uuid', + ]) + ->getMockForAbstractClass(); + + $property_id = uniqid('some-property-'); + $value = uniqid('some-value-'); + $uuid = uniqid('some-uuid-'); + $data = new stdClass(); + $data->identifier = $uuid; + $data->data = $value; + + // Expect. + $mockUuidInterface->expects($this->once()) + ->method('generate') + ->willReturn($uuid); + $mockEntityTypeManager->expects($this->once()) + ->method('getStorage') + ->with('node') + ->willReturn($mockNodeStorage); + $mockNodeStorage->expects($this->once()) + ->method('create') + ->with([ + 'title' => md5(json_encode($value)), + 'type' => 'data', + 'uuid' => $uuid, + 'field_data_type' => $property_id, + 'field_json_metadata' => json_encode($data), + ]) + ->willReturn($mockEntityInterface); + $mockEntityInterface->expects($this->once()) + ->method('save') + ->willReturn(1); + $mockEntityInterface->expects($this->once()) + ->method('uuid') + ->willReturn($uuid); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'createPropertyReference', $property_id, $value); + $this->assertEquals($uuid, $actual); + } + + /** + * Provides data for testing function referenceSingle. + */ + public function dataTestReferenceSingle() { + $property_id = 'some-property'; + $value = 'some-value'; + $uuid = uniqid('existing-reference-uuid-'); + + return [ + 'found an existing reference' => [ + $property_id, $value, $uuid, NULL, $uuid, + ], + 'created a new reference' => [ + $property_id, $value, NULL, $uuid, $uuid, + ], + 'neither found existing nor created new reference' => [ + $property_id, $value, NULL, NULL, $value, + ], + ]; + } + + /** + * Tests function referenceSingle with existing value reference. + * + * @param string $property_id + * The property name. + * @param string $value + * The json value of the property. + * @param string|null $checkExisting + * The expected value of checkExistingReference. + * @param string|null $createProperty + * The expected value of createPropertyReference. + * @param string $expected + * The expected return value of referenceSingle. + * + * @dataProvider dataTestReferenceSingle + */ + public function testReferenceSingle(string $property_id, $value, $checkExisting, $createProperty, $expected) { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(['checkExistingReference', 'createPropertyReference']) + ->getMock(); + + // Expect. + $mock->expects($this->exactly(1)) + ->method('checkExistingReference') + ->with($property_id, $value) + ->willReturn($checkExisting); + $mock->expects($this->any()) + ->method('createPropertyReference') + ->with($property_id, $value) + ->willReturn($createProperty); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'referenceSingle', $property_id, $value); + $this->assertEquals($expected, $actual); + } + + /** + * Test function referenceMultiple. + */ + public function testReferenceMultiple() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(['referenceSingle']) + ->getMock(); + + $property_id = 'theme'; + $values = [ + "Topic One", + "Topic Two", + "Topic Three", + ]; + $referenceSingle = [ + "uuid-one", + "uuid-two", + "uuid-three", + ]; + + // Expect. + $mock->expects($this->exactly(3)) + ->method('referenceSingle') + ->withConsecutive( + [$property_id, $values[0]], + [$property_id, $values[1]], + [$property_id, $values[2]] + ) + ->willReturnOnConsecutiveCalls( + $referenceSingle[0], + $referenceSingle[1], + $referenceSingle[2] + ); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'referenceMultiple', $property_id, $values); + $this->assertEquals($referenceSingle, $actual); + } + + /** + * Provides data for testReferenceProperty. + */ + public function dataTestReferenceProperty() { + $property_id = 'some-property'; + $uuid = uniqid('existing-reference-uuid-'); + + return [ + 'data is a string' => [ + $property_id, + 'some-value', + NULL, + $uuid, + $uuid, + ], + 'data is an object' => [ + $property_id, + (object) ['some' => 'object'], + NULL, + $uuid, + $uuid, + ], + 'data is an array' => [ + $property_id, + ['some value'], + [$uuid], + NULL, + [$uuid], + ], + ]; + } + + /** + * Tests function referenceProperty. + * + * @param string $property_id + * The property name. + * @param string|array $data + * The json value of the property. + * @param string|null $refMultiple + * The expected value of referenceMultiple. + * @param string|null $refSingle + * The expected value of referenceSingle. + * @param string|array $expected + * The expected return value of referenceProperty. + * + * @dataProvider dataTestReferenceProperty + */ + public function testReferenceProperty(string $property_id, $data, $refMultiple, $refSingle, $expected) { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(['referenceMultiple', 'referenceSingle']) + ->getMock(); + + // Expect. + $mock->expects($this->any()) + ->method('referenceMultiple') + ->with($property_id, $data) + ->willReturn($refMultiple); + $mock->expects($this->any()) + ->method('referenceSingle') + ->with($property_id, $data) + ->willReturn($refSingle); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'referenceProperty', $property_id, $data); + $this->assertEquals($expected, $actual); + } + + /** + * Tests function getPropertyList. + */ + public function testGetPropertyList() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(NULL) + ->disableOriginalConstructor() + ->getMock(); + + $mockConfigService = $this->getMockBuilder(ConfigFactoryInterface::class) + ->setMethods(['get']) + ->getMockForAbstractClass(); + + $this->writeProtectedProperty($mock, 'configService', $mockConfigService); + + $mockImmutableConfig = $this->getMockBuilder(ImmutableConfig::class) + ->setMethods(['get']) + ->disableOriginalConstructor() + ->getMock(); + + $this->writeProtectedProperty($mock, 'configService', $mockConfigService); + + $list = "theme + some property + yet another property + + "; + $expected = [ + 'theme', + 'some property', + 'yet another property', + ]; + + // Expect. + $mockConfigService->expects($this->once()) + ->method('get') + ->with('dkan_data.settings') + ->willReturn($mockImmutableConfig); + + $mockImmutableConfig->expects($this->once()) + ->method('get') + ->with('property_list') + ->willReturn($list); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'getPropertyList'); + $this->assertEquals($expected, $actual); + } + + /** + * Tests function reference. + */ + public function testReference() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(['getPropertyList', 'referenceProperty']) + ->disableOriginalConstructor() + ->getMock(); + + $properties_list = [ + 'property to be referenced', + 'another to reference', + ]; + $data = (object) [ + 'property to be referenced' => 'Some value', + 'another to reference' => 'Yet another value', + 'other property' => 'Some other value', + ]; + $uuid1 = uniqid('property1-reference-'); + $uuid2 = uniqid('property2-reference-'); + $expected = (object) [ + 'property to be referenced' => $uuid1, + 'another to reference' => $uuid2, + 'other property' => 'Some other value', + ]; + + // Expect. + $mock->expects($this->once()) + ->method('getPropertyList') + ->willReturn($properties_list); + $mock->expects($this->exactly(2)) + ->method('referenceProperty') + ->withConsecutive( + [$properties_list[0], $data->{$properties_list[0]}], + [$properties_list[1], $data->{$properties_list[1]}] + ) + ->willReturnOnConsecutiveCalls( + $uuid1, + $uuid2 + ); + + // Assert. + $this->assertEquals($expected, $mock->reference($data)); + } + + /** + * Provides data for testDereferenceProperty. + */ + public function dataTestDereferenceProperty() { + $property_id = 'some-property'; + $uuid1 = uniqid('property-one-uuid-'); + $value1_retrieved = uniqid('value-one-retrieved-'); + $uuid2 = uniqid('property-two-uuid-'); + $value2_retrieved = uniqid('value-two-retrieved-'); + + return [ + 'dereferencing a single uuid' => [ + $property_id, + $uuid1, + NULL, + $value1_retrieved, + $value1_retrieved, + ], + 'dereferencing an array of uuid' => [ + $property_id, + [$uuid1, $uuid2], + [$value1_retrieved, $value2_retrieved], + NULL, + [$value1_retrieved, $value2_retrieved], + ], + ]; + } + + /** + * Tests function dereferenceProperty. + * + * @param string $property_id + * The property name. + * @param string|array $uuids + * One or more uuids from the property's value. + * @param string $deRefMultiple + * The expected value of dereferenceMultiple. + * @param string $deRefSingle + * The expected value of dereferenceSingle. + * @param string|array $expected + * The expected return value of dereferenceProperty. + * + * @dataProvider dataTestDereferenceProperty + */ + public function testDereferenceProperty(string $property_id, $uuids, $deRefMultiple, $deRefSingle, $expected) { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(['dereferenceMultiple', 'dereferenceSingle']) + ->getMock(); + + // Expect. + $mock->expects($this->any()) + ->method('dereferenceMultiple') + ->with($property_id, $uuids) + ->willReturn($deRefMultiple); + $mock->expects($this->any()) + ->method('dereferenceSingle') + ->with($property_id, $uuids) + ->willReturn($deRefSingle); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'dereferenceProperty', $property_id, $uuids); + $this->assertEquals($expected, $actual); + } + + /** + * Tests function dereferenceMultiple. + */ + public function testDereferenceMultiple() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(['dereferenceSingle']) + ->getMock(); + + $property_id = 'someProperty'; + $uuids = [ + uniqid('property-uuid1-'), + uniqid('property-uuid2-'), + 'Third, non-referenced property value', + ]; + $dereferenceSingle = [ + "First property value", + "Second property value", + "Third, non-referenced property value", + ]; + + // Expect. + $mock->expects($this->exactly(3)) + ->method('dereferenceSingle') + ->withConsecutive( + [$property_id, $uuids[0]], + [$property_id, $uuids[1]], + [$property_id, $uuids[2]] + ) + ->willReturnOnConsecutiveCalls( + $dereferenceSingle[0], + $dereferenceSingle[1], + $dereferenceSingle[2] + ); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'dereferenceMultiple', $property_id, $uuids); + $this->assertEquals($dereferenceSingle, $actual); + } + + /** + * Provides data for testing checkExistingReference function. + */ + public function datatestDereferenceSingle() { + $mockNode = $this->createMock(NodeInterface::class); + $uuid = uniqid('some-property-uuid-'); + $expected = "Some Property Value"; + $mockNode->field_json_metadata = (object) ['value' => '{"data": "Some Property Value"}']; + + return [ + ['someProperty', $uuid, [$mockNode], $expected], + ['someProperty', $uuid, [], $uuid], + ]; + } + + /** + * Tests function dereferenceSingle. + * + * @param string $property_id + * The property name. + * @param string $uuid + * The uuid. + * @param array $nodes + * The expected $nodes array internally. + * @param string $expected + * The expected return value of dereferenceSingle. + * + * @dataProvider dataTestDereferenceSingle + */ + public function testDereferenceSingle(string $property_id, string $uuid, array $nodes, $expected) { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(NULL) + ->disableOriginalConstructor() + ->getMock(); + $mockEntityTypeManager = $this->getMockBuilder(EntityTypeManagerInterface::class) + ->setMethods([ + 'getStorage', + ]) + ->getMockForAbstractClass(); + $this->writeProtectedProperty($mock, 'entityTypeManager', $mockEntityTypeManager); + $mockNodeStorage = $this->getMockBuilder(EntityStorageInterface::class) + ->setMethods([ + 'loadByProperties', + ]) + ->getMockForAbstractClass(); + + // Expect. + $mockEntityTypeManager->expects($this->once()) + ->method('getStorage') + ->with('node') + ->willReturn($mockNodeStorage); + $mockNodeStorage->expects($this->once()) + ->method('loadByProperties') + ->with([ + 'field_data_type' => $property_id, + 'uuid' => $uuid, + ]) + ->willReturn($nodes); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'dereferenceSingle', $property_id, $uuid); + $this->assertEquals($expected, $actual); + } + + /** + * Tests function dereference. + */ + public function testDereference() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(['getPropertyList', 'dereferenceProperty']) + ->disableOriginalConstructor() + ->getMock(); + + $properties_list = [ + 'property to be dereferenced', + 'another to dereference', + ]; + $uuid1 = uniqid('uuid1-'); + $uuid2 = uniqid('uuid2-'); + $data = (object) [ + 'property to be dereferenced' => $uuid1, + 'another to dereference' => $uuid2, + 'other property' => 'Some other value', + ]; + $expected = (object) [ + 'property to be dereferenced' => 'Some value', + 'another to dereference' => 'Yet another value', + 'other property' => 'Some other value', + ]; + + // Expect. + $mock->expects($this->once()) + ->method('getPropertyList') + ->willReturn($properties_list); + $mock->expects($this->exactly(2)) + ->method('dereferenceProperty') + ->withConsecutive( + [$properties_list[0], $uuid1], + [$properties_list[1], $uuid2] + ) + ->willReturnOnConsecutiveCalls( + 'Some value', + 'Yet another value' + ); + + // Assert. + $this->assertEquals($expected, $mock->dereference($data)); + } + + /** + * Provides data to test emptyPropertyOfSameType. + */ + public function dataEmptyPropertyOfSameType() { + return [ + [ + "some string", "", + ], + [ + ['some', 'array'], [], + ], + ]; + } + + /** + * Tests the emptyPropertyOfSameType function. + * + * @dataProvider dataEmptyPropertyOfSameType + */ + public function testEmptyPropertyOfSameType($data, $expected) { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->disableOriginalConstructor() + ->setMethods(NULL) + ->getMock(); + + // Assert. + $actual = $this->invokeProtectedMethod($mock, 'emptyPropertyOfSameType', $data); + $this->assertEquals($expected, $actual); + } + + /** + * Tests function processReferencesInUpdatedProperty. + */ + public function testProcessReferencesInUpdatedProperty() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(['queueReferenceForRemoval']) + ->disableOriginalConstructor() + ->getMock(); + + $property_id = 'someProperty'; + $old = uniqid('some-uuid-old-'); + $new = uniqid('some-uuid-new-'); + + $mock->expects($this->once()) + ->method('queueReferenceForRemoval') + ->willReturn(NULL); + + // Assert. + $this->invokeProtectedMethod($mock, 'processReferencesInUpdatedProperty', $property_id, $old, $new); + } + + /** + * Tests function processReferencesInDeletedProperty. + */ + public function testProcessReferencesInDeletedProperty() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(['queueReferenceForRemoval']) + ->disableOriginalConstructor() + ->getMock(); + + $property_id = 'someProperty'; + $old = uniqid('some-uuid-old-'); + + $mock->expects($this->once()) + ->method('queueReferenceForRemoval') + ->willReturn(NULL); + + // Assert. + $this->invokeProtectedMethod($mock, 'processReferencesInDeletedProperty', $property_id, $old); + } + + /** + * Tests function processReferencesInDeletedDataset. + */ + public function testProcessReferencesInDeletedDataset() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods(['getPropertyList', 'processReferencesInDeletedProperty']) + ->disableOriginalConstructor() + ->getMock(); + + $properties_list = [ + 'property to be dereferenced', + 'another to dereference', + ]; + $uuid1 = uniqid('uuid1-'); + $uuid2 = uniqid('uuid2-'); + $data = (object) [ + 'property to be dereferenced' => $uuid1, + 'another to dereference' => $uuid2, + 'other property' => 'Some other value', + ]; + + // Expect. + $mock->expects($this->once()) + ->method('getPropertyList') + ->willReturn($properties_list); + $mock->expects($this->exactly(2)) + ->method('processReferencesInDeletedProperty') + ->willReturn(NULL); + + // Assert. + $this->invokeProtectedMethod($mock, 'processReferencesInDeletedDataset', $data); + } + + /** + * Tests function processReferencesInUpdatedDataset. + */ + public function testProcessReferencesInUpdatedDataset() { + // Setup. + $mock = $this->getMockBuilder(ValueReferencer::class) + ->setMethods([ + 'getPropertyList', + 'emptyPropertyOfSameType', + 'processReferencesInUpdatedProperty', + ]) + ->disableOriginalConstructor() + ->getMock(); + + $properties_list = [ + 'property only in old', + 'property only in new', + ]; + $old_uuid = uniqid('uuid-old-'); + $new_uuid = uniqid('uuid-new-'); + $old = (object) [ + 'property only in old' => $old_uuid, + ]; + $new = (object) [ + 'property only in new' => $new_uuid, + ]; + + // Expect. + $mock->expects($this->once()) + ->method('getPropertyList') + ->willReturn($properties_list); + $mock->expects($this->once()) + ->method('processReferencesInUpdatedProperty') + ->willReturn(NULL); + $mock->expects($this->once()) + ->method('emptyPropertyOfSameType') + ->with() + ->willReturn(NULL); + + // Assert. + $this->invokeProtectedMethod($mock, 'processReferencesInUpdatedDataset', $old, $new); + } + +} diff --git a/modules/custom/dkan_harvest/src/Reverter.php b/modules/custom/dkan_harvest/src/Reverter.php index 77eb070451..fe560d2282 100644 --- a/modules/custom/dkan_harvest/src/Reverter.php +++ b/modules/custom/dkan_harvest/src/Reverter.php @@ -41,6 +41,7 @@ public function run() { /** @var \Drupal\dkan_api\Storage\DrupalNodeDataset $datastore_storage */ // Cannot use DI here since this class is called by a non drupal package. $datastore_storage = \Drupal::service('dkan_api.storage.drupal_node_dataset'); + $datastore_storage->setSchema('dataset'); $counter = 0; foreach ($uuids as $uuid) { diff --git a/modules/custom/interra_api/src/Controller/ApiController.php b/modules/custom/interra_api/src/Controller/ApiController.php index 5f8b1fb913..a997ed69c3 100644 --- a/modules/custom/interra_api/src/Controller/ApiController.php +++ b/modules/custom/interra_api/src/Controller/ApiController.php @@ -87,6 +87,7 @@ public function collection($collection) { /** @var \Drupal\dkan_api\Storage\DrupalNodeDataset $storage */ $storage = \Drupal::service('dkan_api.storage.drupal_node_dataset'); + $storage->setSchema('dataset'); $data = $storage->retrieveAll(); if ($collection == "dataset") { @@ -172,6 +173,8 @@ protected function docDatasetHandler($doc) { $uuid = str_replace(".json", "", $doc); /** @var \Drupal\dkan_api\Storage\DrupalNodeDataset $storage */ $storage = \Drupal::service('dkan_api.storage.drupal_node_dataset'); + $storage->setSchema('dataset'); + /** @var \Drupal\interra_api\Service\DatasetModifier $datasetModifuer */ $datasetModifier = \Drupal::service('interra_api.service.dataset_modifier'); $data = $storage->retrieve($uuid); diff --git a/modules/custom/interra_api/tests/src/Unit/Controller/ApiControllerTest.php b/modules/custom/interra_api/tests/src/Unit/Controller/ApiControllerTest.php index 8063f4e2ee..0984415398 100644 --- a/modules/custom/interra_api/tests/src/Unit/Controller/ApiControllerTest.php +++ b/modules/custom/interra_api/tests/src/Unit/Controller/ApiControllerTest.php @@ -378,6 +378,7 @@ public function testDocDatasetHandler() { $mockDrupalNodeDataset = $this->getMockBuilder(Storage::class) ->setMethods([ + 'setSchema', 'retrieve', ]) ->getMockForAbstractClass();