Convert resource map to Drupal Entity API

modules/common/src/DataResource.php

@@ -4,6 +4,7 @@
 
 use Drupal\datastore\DatastoreResource;
 use Drupal\datastore\Service\ResourceLocalizer;
+use Drupal\metastore\ResourceMappingInterface;
 use Procrastinator\JsonSerializeTrait;
 
 /**
@@ -17,10 +18,10 @@
  * Currently, the allowable perspectives are:
  * - 'source' which represents a CSV file somewhere on the internet. In this
  *   case the file path will be an http:// URL.
- * - 'local_path' which represents the same CSV file as a source file, but
+ * - 'local_file' which represents the same CSV file as a source file, but
  *   stored locally in the file system. The file path will be a local URI,
  *   probably in the public:// scheme.
- * - 'local_url' which also can exist.
+ * - 'local_url' which is the 'hostified' version of the local URL to the file.
  *
  * Feed these various permutations of the DataResource object to the
  * ResourceMapper, and it will store these differences in a database table.
@@ -39,6 +40,8 @@
  * For more details refer to the methods governing these behaviors:
  * 1. Resource::createNewVersion()
  * 2. Resource::createNewPerspective()
+ *
+ * @see \Drupal\metastore\Entity\ResourceMapping
  */
 class DataResource implements \JsonSerializable {
 
@@ -123,6 +126,10 @@ public function __construct($file_path, $mimeType, $perspective = self::DEFAULT_
    *
    * @return \Drupal\common\DataResource
    *   DataResource object.
+   *
+   * @deprecated Use DataResource::createFromEntity() instead.
+   *
+   * @see self::createFromEntity()
    */
   public static function createFromRecord(object $record): DataResource {
     $resource = new static($record->filePath, $record->mimeType, $record->perspective);
@@ -133,6 +140,28 @@ public static function createFromRecord(object $record): DataResource {
     return $resource;
   }
 
+  /**
+   * Create a DataResource object from a Drupal entity.
+   *
+   * @param \Drupal\metastore\ResourceMappingInterface $mapping
+   *   A resource_mapping entity.
+   *
+   * @return \Drupal\common\DataResource
+   *   DataResource object.
+   */
+  public static function createFromEntity(ResourceMappingInterface $mapping): DataResource {
+    $resource = new static(
+      $mapping->get('filePath')->getString(),
+      $mapping->get('mimeType')->getString(),
+      $mapping->get('perspective')->getString()
+    );
+    // MD5 of record's file path can differ from the MD5 generated in the
+    // constructor, so we have to explicitly set the identifier.
+    $resource->identifier = $mapping->get('identifier')->getString();
+    $resource->version = $mapping->get('version')->getString();
+    return $resource;
+  }
+
   /**
    * Clone the current resource with a new version identifier.
    *
@@ -243,6 +272,17 @@ public function getPerspective() {
     return $this->perspective;
   }
 
+  /**
+   * Get the checksum for this resource.
+   *
+   * @return string|null
+   *   The checksum for the local file perspective. NULL if it has not yet been
+   *   computed, or if this resource is a different perspective.
+   */
+  public function getChecksum(): ?string {
+    return $this->checksum;
+  }
+
   /**
    * Get a unique identifier for this data resource.
    *

modules/datastore/tests/src/Functional/ImportLocalCopyOfRemoteFileTest.php

@@ -4,6 +4,7 @@
 
 use Drupal\common\DataResource;
 use Drupal\common\FileFetcher\FileFetcherRemoteUseExisting;
+use Drupal\Core\Entity\EntityStorageInterface;
 use Drupal\datastore\Service\ResourceLocalizer;
 use Drupal\Tests\BrowserTestBase;
 use FileFetcher\FileFetcher;
@@ -45,11 +46,14 @@ public function test() {
 
     $identifier = uniqid();
 
+    // Get mapping entity storage.
+    /** @var \Drupal\Core\Entity\EntityStorageInterface $mapping_entity_storage */
+    $mapping_entity_storage = $this->container
+      ->get('entity_type.manager')
+      ->getStorage('resource_mapping');
+
     // There should be no mapper records.
-    /** @var \Drupal\metastore\ResourceMapper $resource_mapper */
-    $resource_mapper = $this->container->get('dkan.metastore.resource_mapper');
-    $mapping_store = $resource_mapper->getStore();
-    $this->assertEquals(0, $mapping_store->count());
+    $this->assertEquals(0, $this->getEntityCount($mapping_entity_storage));
 
     // Post our dataset.
     /** @var \Drupal\metastore\MetastoreService $metastore_service */
@@ -63,25 +67,28 @@ public function test() {
     );
 
     // 1 mapping after posting the datastore.
-    $this->assertEquals(1, $mapping_store->count());
+    $this->assertEquals(1, $this->getEntityCount($mapping_entity_storage));
 
     // Get our resource info from the dataset info service.
     /** @var \Drupal\common\DatasetInfo $dataset_info_service */
     $dataset_info_service = $this->container->get('dkan.common.dataset_info');
     $info = $dataset_info_service->gather($identifier);
 
     // Having gotten the info, there should still only be 1 record.
-    $this->assertEquals(1, $mapping_store->count());
+    $this->assertEquals(1, $this->getEntityCount($mapping_entity_storage));
 
     // Let's interrogate it. It should be a source mapping.
-    $all_mappings = $mapping_store->retrieveAll();
-    $mapping = $mapping_store->retrieve(reset($all_mappings));
+    $all_mapping_ids = $this->getAllEntityIds($mapping_entity_storage);
+    /** @var \Drupal\Core\Entity\EntityInterface $mapping */
+    $mapping = $mapping_entity_storage->load(reset($all_mapping_ids));
     $this->assertEquals(
       DataResource::DEFAULT_SOURCE_PERSPECTIVE,
-      $mapping->perspective
+      $mapping->get('perspective')->getString()
     );
 
     // Now let's ask the mapper instead of its storage.
+    /** @var \Drupal\metastore\ResourceMapper $resource_mapper */
+    $resource_mapper = $this->container->get('dkan.metastore.resource_mapper');
     $this->assertNotNull(
       $resource_id = $info['latest_revision']['distributions'][0]['resource_id'] ?? NULL
     );
@@ -90,7 +97,7 @@ public function test() {
       $source_resource = $resource_mapper->get($resource_id)
     );
     // Getting the source resource should not change the record count.
-    $this->assertEquals(1, $mapping_store->count());
+    $this->assertEquals(1, $this->getEntityCount($mapping_entity_storage));
     // No local file perspective yet.
     $this->assertNull($resource_mapper->get(
       $resource_id,
@@ -157,7 +164,7 @@ public function test() {
     );
 
     // Now there should be three mappings.
-    $this->assertEquals(3, $mapping_store->count());
+    $this->assertEquals(3, $this->getEntityCount($mapping_entity_storage));
     // Get the info again after localization.
     $localized_info = $dataset_info_service->gather($identifier);
     $this->assertNotEquals($localized_info, $info);
@@ -167,6 +174,37 @@ public function test() {
     );
   }
 
+  /**
+   * Count the number of entities present for the given storage.
+   *
+   * @param \Drupal\Core\Entity\EntityStorageInterface $storage
+   *   The entity storage object.
+   *
+   * @return int
+   *   The count of arrays present in the storage.
+   */
+  protected function getEntityCount(EntityStorageInterface $storage) {
+    return $storage->getQuery()
+      ->count()
+      ->accessCheck(FALSE)
+      ->execute();
+  }
+
+  /**
+   * Get all the IDs available for the given entity storage.
+   *
+   * @param \Drupal\Core\Entity\EntityStorageInterface $storage
+   *   The entity storage object.
+   *
+   * @return array
+   *   Array of entity IDs for all the entities in the storage.
+   */
+  protected function getAllEntityIds(EntityStorageInterface $storage) {
+    return $storage->getQuery()
+      ->accessCheck(FALSE)
+      ->execute();
+  }
+
   /**
    * Generate dataset metadata, possibly with multiple distributions.
    *

modules/datastore/tests/src/Kernel/Plugin/QueueWorker/ImportQueueWorkerTest.php

@@ -29,6 +29,8 @@ class ImportQueueWorkerTest extends KernelTestBase {
   ];
 
   public function testErrorPath() {
+    $this->installEntitySchema('resource_mapping');
+
     // The result we'll mock to come from the datastore service.
     $result = new Result();
     $result->setStatus(Result::ERROR);
@@ -79,6 +81,7 @@ public function testErrorPath() {
   }
 
   public function testRequeue() {
+    $this->installEntitySchema('resource_mapping');
     // The result we'll mock to come from the datastore service.
     $result = new Result();
     $result->setStatus(Result::STOPPED);
@@ -166,6 +169,7 @@ public function testProcessItemAlreadyImported() {
    * @covers ::processItem
    */
   public function testProcessItemImportException() {
+    $this->installEntitySchema('resource_mapping');
     // Mock the logger so we can tell when the error occurs.
     $logger = $this->getMockForAbstractClass(LoggerChannelInterface::class);
     // We expect an error to be logged, and we set an expectation for the message.

modules/datastore/tests/src/Kernel/Plugin/QueueWorker/LocalizeQueueWorkerTest.php

@@ -34,6 +34,11 @@ class LocalizeQueueWorkerTest extends KernelTestBase {
     'node',
   ];
 
+  protected function setUp() : void {
+    parent::setUp();
+    $this->installEntitySchema('resource_mapping');
+  }
+
   public function provideUseExisting() {
     return [
       'Use existing localized file' => [TRUE],

modules/datastore/tests/src/Kernel/Service/Info/ImportInfoTest.php

@@ -33,6 +33,11 @@ class ImportInfoTest extends KernelTestBase {
 
   protected const SOURCE_URL = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit/district_centerpoints_small.csv';
 
+  protected function setUp() : void {
+    parent::setUp();
+    $this->installEntitySchema('resource_mapping');
+  }
+
   public function testFileSize() {
     $source_resource = new DataResource(
       self::SOURCE_URL,

modules/datastore/tests/src/Kernel/Service/ResourceLocalizerTest.php

@@ -35,6 +35,12 @@ class ResourceLocalizerTest extends KernelTestBase {
 
   protected const SOURCE_URL = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit/district_centerpoints_small.csv';
 
+  protected function setUp() : void {
+    parent::setUp();
+    // All our tests need the resource_mapping entity.
+    $this->installEntitySchema('resource_mapping');
+  }
+
   public function testNoResourceFound() {
     $service = $this->container->get('dkan.datastore.service.resource_localizer');
 

modules/datastore/tests/src/Kernel/UseLocalWithPrepareLocalizeTest.php

@@ -31,6 +31,11 @@ class UseLocalWithPrepareLocalizeTest extends KernelTestBase {
 
   protected const SOURCE_URL = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit/district_centerpoints_small.csv';
 
+  protected function setUp() : void {
+    parent::setUp();
+    $this->installEntitySchema('resource_mapping');
+  }
+
   public function test() {
     $this->installConfig(['common']);
 

modules/metastore/metastore.install

@@ -114,3 +114,16 @@ function metastore_update_8007() {
   $config = \Drupal::service('config.factory')->getEditable('node.type.data');
   $config->set('preview_mode', 1)->save();
 }
+
+/**
+ * Ensure the entity manager knows about resource_mapping entities.
+ *
+ * Note: The dkan_metastore_resource_mapper table might already exist in your
+ * database. The resource_mapping entity is designed to use it without
+ * modification.
+ */
+function metastore_update_8008(&$sandbox) {
+  \Drupal::entityTypeManager()->clearCachedDefinitions();
+  \Drupal::entityDefinitionUpdateManager()
+    ->installEntityType(\Drupal::entityTypeManager()->getDefinition('resource_mapping'));
+}

modules/metastore/metastore.permissions.yml

@@ -6,3 +6,7 @@
   description: 'Administer data dictionary settings'
 
 
+
+administer resource mapping:
+  title: 'Administer resource mapping settings'
+  restrict access: true

modules/metastore/metastore.services.yml

@@ -63,7 +63,7 @@ services:
     class: \Drupal\metastore\ResourceMapper
     arguments:
       - '@dkan.metastore.resource_mapper_database_table'
-      - '@event_dispatcher'
+      - '@entity_type.manager'
 
   dkan.metastore.resource_mapper_database_table:
     class: \Drupal\metastore\Storage\ResourceMapperDatabaseTable