From 48ce39c8c43cb892d2f486c7c117a793c7744382 Mon Sep 17 00:00:00 2001 From: Dan Feder Date: Fri, 12 Apr 2024 11:40:06 -0500 Subject: [PATCH] Support choosing data dictionaries in dataset node form (#4067) --- cypress/support/helpers/dkan.js | 2 +- .../user-guide/guide_data_dictionaries.rst | 2 +- .../common/tests/src/Traits/GetDataTrait.php | 2 +- .../json_form_widget/src/SchemaUiHandler.php | 36 ++- modules/json_form_widget/src/WidgetRouter.php | 107 +++++---- .../tests/src/Unit/WidgetRouterTest.php | 209 ++++++++++++++++++ modules/metastore/metastore.install | 23 ++ modules/metastore/src/LifeCycle/LifeCycle.php | 15 +- modules/metastore/src/NodeWrapper/Data.php | 10 + .../Api1/DistributionHandlingTest.php | 4 +- .../Kernel/DataDictionarySettingsFormTest.php | 2 +- .../tests/src/files/DataDictionary.json | 2 +- schema/collections/data-dictionary.json | 12 +- 13 files changed, 362 insertions(+), 64 deletions(-) create mode 100644 modules/json_form_widget/tests/src/Unit/WidgetRouterTest.php diff --git a/cypress/support/helpers/dkan.js b/cypress/support/helpers/dkan.js index 764df8d850..778af9528f 100644 --- a/cypress/support/helpers/dkan.js +++ b/cypress/support/helpers/dkan.js @@ -191,8 +191,8 @@ export function generateKeyword(uuid) { export function generateDataDictionary(uuid) { return { "identifier": uuid, - "title": "Title for " + uuid, "data": { + "title": "Title for " + uuid, "fields": [ { "name": generateRandomString(), diff --git a/docs/source/user-guide/guide_data_dictionaries.rst b/docs/source/user-guide/guide_data_dictionaries.rst index ed09e62944..169c8bf3eb 100644 --- a/docs/source/user-guide/guide_data_dictionaries.rst +++ b/docs/source/user-guide/guide_data_dictionaries.rst @@ -130,8 +130,8 @@ We will define a list of fields based on the example header row below. Authorization: Basic username:password { - "title": "Demo Dictionary", "data": { + "title": "Demo Dictionary", "fields": [ { "name": "project_id", diff --git a/modules/common/tests/src/Traits/GetDataTrait.php b/modules/common/tests/src/Traits/GetDataTrait.php index cd01cf625f..65491339ce 100644 --- a/modules/common/tests/src/Traits/GetDataTrait.php +++ b/modules/common/tests/src/Traits/GetDataTrait.php @@ -89,8 +89,8 @@ private function getDataset(string $identifier, string $title, array $downloadUr private function getDataDictionary(array $fields, array $indexes, string $identifier, string $title = 'Test DataDict') { return json_encode([ 'identifier' => $identifier, - 'title' => $title, 'data' => [ + 'title' => $title, 'fields' => $fields, 'indexes' => $indexes, ], diff --git a/modules/json_form_widget/src/SchemaUiHandler.php b/modules/json_form_widget/src/SchemaUiHandler.php index 76c0b3220f..9f8462667e 100644 --- a/modules/json_form_widget/src/SchemaUiHandler.php +++ b/modules/json_form_widget/src/SchemaUiHandler.php @@ -173,12 +173,46 @@ public function applyOnBaseField($spec, array $element) { $element = $this->changeFieldDescriptions($spec->{"ui:options"}, $element); $element = $this->changeFieldTitle($spec->{"ui:options"}, $element); if (isset($spec->{"ui:options"}->hideActions)) { - $element = $this->widgetRouter->flattenArrays($spec->{"ui:options"}, $element); + $element = $this->flattenArrays($spec->{"ui:options"}, $element); } } return $element; } + /** + * Flatten array elements and unset actions if hideActions is set. + * + * @param mixed $spec + * Object with spec for UI options. + * @param array $element + * Element to apply UI options. + * + * @return array + * Return flattened element without actions. + */ + public function flattenArrays($spec, array $element) { + unset($element['actions']); + $default_value = []; + foreach ($element[$spec->child] as $key => $item) { + $default_value = array_merge($default_value, $this->formatArrayDefaultValue($item)); + if ($key != 0) { + unset($element[$spec->child][$key]); + } + } + $element[$spec->child][0]['#default_value'] = $default_value; + return $element; + } + + /** + * Format default values for arrays (flattened). + */ + private function formatArrayDefaultValue($item) { + if (!empty($item['#default_value'])) { + return [$item['#default_value'] => $item['#default_value']]; + } + return []; + } + /** * Apply schema UI to object fields. * diff --git a/modules/json_form_widget/src/WidgetRouter.php b/modules/json_form_widget/src/WidgetRouter.php index 0193018400..c6765f4936 100644 --- a/modules/json_form_widget/src/WidgetRouter.php +++ b/modules/json_form_widget/src/WidgetRouter.php @@ -102,40 +102,6 @@ public function getWidgets() { ]; } - /** - * Flatten array elements and unset actions if hideActions is set. - * - * @param mixed $spec - * Object with spec for UI options. - * @param array $element - * Element to apply UI options. - * - * @return array - * Return flattened element without actions. - */ - public function flattenArrays($spec, array $element) { - unset($element['actions']); - $default_value = []; - foreach ($element[$spec->child] as $key => $item) { - $default_value = array_merge($default_value, $this->formatArrayDefaultValue($item)); - if ($key != 0) { - unset($element[$spec->child][$key]); - } - } - $element[$spec->child][0]['#default_value'] = $default_value; - return $element; - } - - /** - * Format default values for arrays (flattened). - */ - private function formatArrayDefaultValue($item) { - if (!empty($item['#default_value'])) { - return [$item['#default_value'] => $item['#default_value']]; - } - return []; - } - /** * Handle configuration for list elements. * @@ -148,14 +114,19 @@ private function formatArrayDefaultValue($item) { * The element configured as a list element. */ public function handleListElement($spec, array $element) { - if (isset($spec->titleProperty)) { - if (isset($element[$spec->titleProperty])) { - $element[$spec->titleProperty] = $this->getDropdownElement($element[$spec->titleProperty], $spec, $spec->titleProperty); - } + $title_property = ($spec->titleProperty ?? FALSE); + + if (isset($title_property, $element[$title_property])) { + $element[$title_property] = $this->getDropdownElement($element[$title_property], $spec, $title_property); } - else { + + if (isset($spec->source->returnValue)) { + $element = $this->getDropdownElement($element, $spec, $title_property); + } + elseif (!isset($spec->titleProperty)) { $element = $this->getDropdownElement($element, $spec); } + return $element; } @@ -245,19 +216,59 @@ public function getDropdownOptions($source, $titleProperty = FALSE) { */ public function getOptionsFromMetastore($source, $titleProperty = FALSE) { $options = []; - $values = $this->metastore->getAll($source->metastoreSchema); - foreach ($values as $value) { - $value = json_decode($value); - if ($titleProperty) { - $options[$value->data->{$titleProperty}] = $value->data->{$titleProperty}; - } - else { - $options[$value->data] = $value->data; - } + $metastore_items = $this->metastore->getAll($source->metastoreSchema); + foreach ($metastore_items as $item) { + $item = json_decode($item); + $title = $this->metastoreOptionTitle($item, $source, $titleProperty); + $value = $this->metastoreOptionValue($item, $source, $titleProperty); + $options[$value] = $title; } return $options; } + /** + * Determine the title for the select option. + * + * @param object|string $item + * Single item from Metastore::getAll() + * @param object $source + * Source defintion from UI schema. + * @param string|false $titleProperty + * Title property defined in UI schema. + * + * @return string + * String to be used in title. + */ + private function metastoreOptionTitle($item, object $source, $titleProperty): string { + if ($titleProperty) { + return is_object($item) ? $item->data->$titleProperty : $item; + } + return $item->data; + } + + /** + * Determine the value for the select option. + * + * @param object|string $item + * Single item from Metastore::getAll() + * @param object $source + * Source defintion from UI schema. + * @param string|false $titleProperty + * Title property defined in UI schema. + * + * @return string + * String to be used as option value. + */ + private function metastoreOptionValue($item, object $source, $titleProperty): string { + if (($source->returnValue ?? NULL) == 'url') { + return 'dkan://metastore/schemas/' . $source->metastoreSchema . '/items/' . $item->identifier; + } + if ($titleProperty) { + return is_object($item) ? $item->data->$titleProperty : $item; + } + return $item->data; + } + /** * Helper function to add the value of other to current list of options. */ diff --git a/modules/json_form_widget/tests/src/Unit/WidgetRouterTest.php b/modules/json_form_widget/tests/src/Unit/WidgetRouterTest.php new file mode 100644 index 0000000000..f1a839720c --- /dev/null +++ b/modules/json_form_widget/tests/src/Unit/WidgetRouterTest.php @@ -0,0 +1,209 @@ +getContainerChain()->getMock()); + + $new_element = $router->getConfiguredWidget($spec, $element); + + $this->assertEquals($handledElement, $new_element); + } + + private function getContainerChain() { + $containerGetOptions = (new Options()) + ->add('uuid', Php::class) + ->add('json_form.string_helper', StringHelper::class) + ->add('dkan.metastore.service', MetastoreService::class) + ->index(0); + + $metastoreGetAllOptions = (new Options()) + ->add('publisher', self::publishers()) + ->add('data-dictionary', self::dataDictionaries()) + ->index(0); + + return (new Chain($this)) + ->add(Container::class, 'get', $containerGetOptions) + ->add(MetastoreService::class, 'getAll', $metastoreGetAllOptions); + } + + public static function dataProvider(): array { + return [ + // Tag field is a free-tagging autocomplete that populates from metastore. + 'tagField' => [ + (object) [ + 'widget' => 'list', + 'type' => 'autocomplete', + 'allowComplete' => TRUE, + 'multiple' => TRUE, + 'source' => [ + 'metastoreSchema' => 'theme', + ], + ], + [ + '#type' => 'textfield', + '#title' => 'tags', + ], + [ + '#type' => 'select2', + '#title' => 'tags', + '#options' => [], + '#other_option' => FALSE, + '#multiple' => TRUE, + '#autocreate' => FALSE, + '#target_type' => 'node', + ], + ], + // Format is a simple select field with values defined in UI schema. + 'formatField' => [ + (object) [ + "title" => "File Format", + "widget" => "list", + "type" => "select_other", + "other_type" => "textfield", + "source" => (object) [ + "enum" => [ + "csv", + "json", + ], + ], + ], + [ + '#type' => 'textfield', + '#title' => 'File Format', + ], + [ + '#type' => 'select_or_other_select', + '#title' => 'File Format', + '#options' => [ + 'csv' => 'csv', + 'json' => 'json', + ], + '#other_option' => FALSE, + '#input_type' => 'textfield', + ], + ], + // Publisher popualtes from metastore but returns whole object, + // is wrapped in a details element. + 'publisherField' => [ + (object) [ + "widget" => "list", + "type" => "autocomplete", + "allowCreate" => TRUE, + "titleProperty" => "name", + "source" => (object) [ + "metastoreSchema" => "publisher", + ], + ], + [ + '#type' => 'details', + '#title' => 'Organization', + 'name' => [ + '#type' => 'textfield', + '#title' => "Publisher Name", + "#default_value" => NULL, + "#required" => TRUE, + ], + ], + [ + '#type' => 'details', + '#title' => 'Organization', + 'name' => [ + '#type' => 'select2', + '#title' => 'Publisher Name', + '#default_value' => NULL, + '#required' => TRUE, + '#options' => [ + 'Publisher 1' => 'Publisher 1', + 'Publisher 2' => 'Publisher 2', + ], + '#other_option' => FALSE, + '#multiple' => FALSE, + '#autocreate' => TRUE, + '#target_type' => 'node', + ], + ], + ], + // Data dict field draws from metastore but just shows URLs. + 'dataDict' => [ + (object) [ + "widget" => "list", + "type" => "select", + "titleProperty" => "title", + "source" => (object) [ + "metastoreSchema" => "data-dictionary", + "returnValue" => "url", + ], + ], + [ + '#type' => 'url', + '#title' => 'Data Dictionary', + ], + [ + '#type' => 'select', + '#title' => 'Data Dictionary', + '#options' => [ + 'dkan://metastore/schemas/data-dictionary/items/111' => 'Data dictionary 1', + 'dkan://metastore/schemas/data-dictionary/items/222' => 'Data dictionary 2', + ], + '#other_option' => FALSE, + ], + ], + ]; + } + + public static function publishers() { + return [ + json_encode((object) [ + 'identifier' => '111', + 'data' => (object) [ + '@type' => 'org:Organization', + 'name' => 'Publisher 1', + ], + ]), + json_encode((object) [ + 'identifier' => '222', + 'data' => (object) [ + '@type' => 'org:Organization', + 'name' => 'Publisher 2', + ], + ]), + ]; + } + + public static function dataDictionaries() { + return [ + json_encode((object) [ + 'identifier' => '111', + 'data' => (object) [ + 'title' => 'Data dictionary 1', + ], + ]), + json_encode((object) [ + 'identifier' => '222', + 'data' => (object) [ + 'title' => 'Data dictionary 2', + ], + ]), + ]; + } + + +} diff --git a/modules/metastore/metastore.install b/modules/metastore/metastore.install index 0bfc1465b9..ed57dece1e 100644 --- a/modules/metastore/metastore.install +++ b/modules/metastore/metastore.install @@ -127,3 +127,26 @@ function metastore_update_8008(&$sandbox) { \Drupal::entityDefinitionUpdateManager() ->installEntityType(\Drupal::entityTypeManager()->getDefinition('resource_mapping')); } + +/** + * Update existing data dictionary nodes to use corrected schema. + */ +function metastore_update_8009() { + $ids = \Drupal::service('dkan.metastore.service')->getIdentifiers('data-dictionary'); + $count = 0; + foreach ($ids as $id) { + $dict = \Drupal::service('dkan.metastore.metastore_item_factory')->getInstance($id); + $metadata = $dict->getMetadata(); + if (!isset($metadata->data->title) && isset($metadata->title)) { + $metadata->data->title = $metadata->title; + unset($metadata->title); + } + $dict->setMetadata($metadata); + $dict->save(); + $count++; + } + return t("Updated $count dictionaries. If you have overridden DKAN's core schemas, + you must update your site's data dictionary schema after this update. Copy + modules/contrib/dkan/schema/collections/data-dictionary.json over you local + site version before attempting to read or write any data dictionaries."); +} diff --git a/modules/metastore/src/LifeCycle/LifeCycle.php b/modules/metastore/src/LifeCycle/LifeCycle.php index 280621c0c7..0b74cd8cb2 100644 --- a/modules/metastore/src/LifeCycle/LifeCycle.php +++ b/modules/metastore/src/LifeCycle/LifeCycle.php @@ -400,7 +400,20 @@ protected function setNodeValuesFromMetadata(MetastoreItemInterface $data): void * Data-Dictionary metastore item. */ protected function datadictionaryPresave(MetastoreItemInterface $data): void { - $this->setNodeValuesFromMetadata($data); + $metadata = $data->getMetaData(); + + $title = $metadata->data->title; + $data->setTitle($title); + + // If there is no uuid add one. + if (!isset($metadata->identifier)) { + $metadata->identifier = $data->getIdentifier(); + } + // If one exists in the uuid it should be the same in the table. + else { + $data->setIdentifier($metadata->identifier); + } + $data->setMetadata($metadata); } /** diff --git a/modules/metastore/src/NodeWrapper/Data.php b/modules/metastore/src/NodeWrapper/Data.php index 905e41297c..f83ef40923 100644 --- a/modules/metastore/src/NodeWrapper/Data.php +++ b/modules/metastore/src/NodeWrapper/Data.php @@ -289,4 +289,14 @@ public function getOriginal() { } } + /** + * Save the "wrapped" node. + * + * Useful for some operations - usually recommended to use the metastore + * service's POST and PUT functions rather than saving the node directly. + */ + public function save() { + $this->node->save(); + } + } diff --git a/modules/metastore/tests/src/Functional/Api1/DistributionHandlingTest.php b/modules/metastore/tests/src/Functional/Api1/DistributionHandlingTest.php index 56676f0468..2868ef64ff 100644 --- a/modules/metastore/tests/src/Functional/Api1/DistributionHandlingTest.php +++ b/modules/metastore/tests/src/Functional/Api1/DistributionHandlingTest.php @@ -6,8 +6,6 @@ use Drupal\Tests\common\Functional\Api1TestBase; use GuzzleHttp\RequestOptions; -use function PHPUnit\Framework\assertEquals; - class DistributionHandlingTest extends Api1TestBase { public function getEndpoint():string { @@ -101,7 +99,7 @@ private function postDataDictionary() { $this->assertJsonIsValid($responseSchema, $responseBody); // Unless JSON changes, we should always get same id back. - assertEquals($responseBody->identifier, "5a0a82d9-9a91-5165-b948-927387029590"); + $this->assertEquals("47f1d697-f469-5b41-a613-80cdfac7a326", $responseBody->identifier); return $responseBody->identifier; } diff --git a/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php b/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php index 86c15a5921..9cbfd5a676 100644 --- a/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php +++ b/modules/metastore/tests/src/Kernel/DataDictionarySettingsFormTest.php @@ -100,8 +100,8 @@ public function provideFormData(): array { private function getDataDictionary(array $fields, array $indexes, string $identifier, string $title = 'Test DataDict') { return json_encode([ 'identifier' => $identifier, - 'title' => $title, 'data' => [ + 'title' => $title, 'fields' => $fields, 'indexes' => $indexes, ], diff --git a/modules/metastore/tests/src/files/DataDictionary.json b/modules/metastore/tests/src/files/DataDictionary.json index 4466177a24..e6ed96f47e 100644 --- a/modules/metastore/tests/src/files/DataDictionary.json +++ b/modules/metastore/tests/src/files/DataDictionary.json @@ -1,6 +1,6 @@ { - "title": "Bike lanes data dictionary", "data": { + "title": "Bike lanes data dictionary", "fields": [ { "name": "objectid", diff --git a/schema/collections/data-dictionary.json b/schema/collections/data-dictionary.json index 31c9bb7193..f5f0bdfa8c 100644 --- a/schema/collections/data-dictionary.json +++ b/schema/collections/data-dictionary.json @@ -4,25 +4,25 @@ "type": "object", "required": [ "identifier", - "title" + "data" ], "properties": { "identifier": { "title": "Identifier", "type": "string" }, - "title": { - "title": "Title", - "type": "string" - }, "data": { "title": "Project Open Data Data-Dictionary", "description": "A data dictionary for this resource, compliant with the [Table Schema](https://specs.frictionlessdata.io/table-schema/) specification.", "type": "object", "required": [ - "fields" + "title" ], "properties": { + "title": { + "title": "Title", + "type": "string" + }, "fields": { "title": "Dictionary Fields", "type": "array",