From f70a390cde1fc27e27ac21007ef66f6946f9ba61 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 19 Apr 2023 17:05:46 -0500
Subject: [PATCH 01/23] early work

---
 modules/datastore/src/SqlEndpoint/Service.php | 12 +----
 .../src/Functional/DictionaryEnforcerTest.php |  1 +
 .../Functional/Service/ResourcePurgerTest.php |  1 +
 .../Unit/Plugin/QueueWorker/ImportJobTest.php |  1 +
 modules/harvest/src/Load/Dataset.php          | 30 ++++-------
 modules/harvest/src/Service.php               |  2 -
 .../harvest/tests/src/Unit/ServiceTest.php    |  1 +
 .../Action/HideCurrentRevisionActionTest.php  |  1 +
 .../src/Functional/Api1/DatasetItemTest.php   |  5 +-
 .../Functional/MetastoreApiPageCacheTest.php  |  1 +
 phpunit.xml                                   |  1 +
 tests/src/Functional/DatasetTest.php          | 51 +++++++++----------
 12 files changed, 48 insertions(+), 59 deletions(-)

diff --git a/modules/datastore/src/SqlEndpoint/Service.php b/modules/datastore/src/SqlEndpoint/Service.php
index c8c7ebd586..9cd4027321 100644
--- a/modules/datastore/src/SqlEndpoint/Service.php
+++ b/modules/datastore/src/SqlEndpoint/Service.php
@@ -17,7 +17,7 @@
 /**
  * SQL endpoint service.
  */
-class Service implements ContainerInjectionInterface {
+class Service {
   /**
    * ConfigFactory object.
    *
@@ -32,16 +32,6 @@ class Service implements ContainerInjectionInterface {
    */
   private $datastoreService;
 
-  /**
-   * {@inheritdoc}
-   */
-  public static function create(ContainerInterface $container) {
-    return new static(
-      $container->get('dkan.datastore.service'),
-      $container->get('config.factory')
-    );
-  }
-
   /**
    * Constructor, sets the datastoreService and configFactory properties.
    *
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 9b7b13ba61..7e9b080a94 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,6 +18,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
+ * @coversNothing
  */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index 667f552440..8768a95081 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,6 +13,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
+ * @coversNothing
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index e5207f3dc4..07a43d95fc 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -174,6 +174,7 @@ public function testSerialization() {
    *
    */
   public function testMultiplePasses() {
+    $this->markTestSkipped('flaky test');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();
diff --git a/modules/harvest/src/Load/Dataset.php b/modules/harvest/src/Load/Dataset.php
index 0819641d65..7c5e736c76 100644
--- a/modules/harvest/src/Load/Dataset.php
+++ b/modules/harvest/src/Load/Dataset.php
@@ -11,44 +11,36 @@
  */
 class Dataset extends Load {
 
+  protected Service $metastoreService;
+
+  public function __construct($harvest_plan, $hash_storage, $item_storage) {
+    $this->metastoreService = \Drupal::service('dkan.metastore.service');
+    parent::__construct($harvest_plan, $hash_storage, $item_storage);
+  }
+
   /**
    * Public.
    */
   public function removeItem($id) {
-    $service = $this->getMetastoreService();
-    $service->delete("dataset", "{$id}");
+    $this->metastoreService->delete("dataset", "{$id}");
   }
 
   /**
    * Private.
    */
   protected function saveItem($item) {
-    $service = $this->getMetastoreService();
     if (!is_string($item)) {
       $item = json_encode($item);
     }
 
     $schema_id = 'dataset';
-    $item = $service->getValidMetadataFactory()->get($item, $schema_id);
+    $item = $this->metastoreService->getValidMetadataFactory()->get($item, $schema_id);
     try {
-      $service->post($schema_id, $item);
+      $this->metastoreService->post($schema_id, $item);
     }
     catch (ExistingObjectException $e) {
-      $service->put($schema_id, $item->{"$.identifier"}, $item);
+      $this->metastoreService->put($schema_id, $item->{"$.identifier"}, $item);
     }
   }
 
-  /**
-   * Get the metastore service.
-   *
-   * @return \Drupal\metastore\Service
-   *   Metastore service.
-   *
-   * @codeCoverageIgnore
-   */
-  protected function getMetastoreService(): Service {
-    $service = \Drupal::service('dkan.metastore.service');
-    return $service;
-  }
-
 }
diff --git a/modules/harvest/src/Service.php b/modules/harvest/src/Service.php
index fe49dfa49e..8ac8438390 100644
--- a/modules/harvest/src/Service.php
+++ b/modules/harvest/src/Service.php
@@ -339,8 +339,6 @@ private function getHarvester(string $id) {
 
   /**
    * Protected.
-   *
-   * @codeCoverageIgnore
    */
   protected function getDkanHarvesterInstance($harvestPlan, $item_store, $hash_store) {
     return new DkanHarvester(new Factory($harvestPlan, $item_store, $hash_store));
diff --git a/modules/harvest/tests/src/Unit/ServiceTest.php b/modules/harvest/tests/src/Unit/ServiceTest.php
index 35766cc7f3..8bd4bf68bd 100644
--- a/modules/harvest/tests/src/Unit/ServiceTest.php
+++ b/modules/harvest/tests/src/Unit/ServiceTest.php
@@ -24,6 +24,7 @@
 
 /**
  * @coversDefaultClass \Drupal\harvest\Service
+ * @covers \Drupal\harvest\Service
  * @group harvest
  */
 class ServiceTest extends TestCase {
diff --git a/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php b/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
index 7c536d2878..84bf80cbf5 100644
--- a/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
+++ b/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
@@ -18,6 +18,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @coversNothing
  */
 class HideCurrentRevisionActionTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
index 58da82893d..1c773ffbd4 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
@@ -5,6 +5,9 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 use GuzzleHttp\RequestOptions;
 
+/**
+ * @coversNothing
+ */
 class DatasetItemTest extends Api1TestBase {
 
   public function getEndpoint():string {
@@ -131,4 +134,4 @@ private function assertDatasetGet($dataset) {
     $this->assertEquals($dataset, $responseBody);
   }
 
-}
\ No newline at end of file
+}
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index df5ecaaa16..f6ba614758 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -15,6 +15,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @coversNothing
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/phpunit.xml b/phpunit.xml
index bdeb809091..e1e33203b3 100644
--- a/phpunit.xml
+++ b/phpunit.xml
@@ -12,6 +12,7 @@
       <directory>.</directory>
     </include>
     <exclude>
+      <file>rector.php</file>
       <directory>modules/common/tests</directory>
       <directory>modules/datastore/tests</directory>
       <directory>modules/dkan_js_frontend/tests</directory>
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 35eac47e8c..af7663fab5 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -22,6 +22,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @coversNothing
  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;
@@ -65,7 +66,7 @@ public function testChangingDatasetResourcePerspectiveOnOutput() {
 
     $this->changeDatasetsResourceOutputPerspective(ResourceLocalizer::LOCAL_URL_PERSPECTIVE);
 
-    $metadata = $this->getMetastore()->get('dataset', 123);
+    $metadata = \Drupal::service('dkan.metastore.service')->get('dataset', 123);
     $dataset = json_decode($metadata);
 
     $this->assertNotEquals(
@@ -93,7 +94,11 @@ public function testResourcePurgePublished() {
    * Test the resource purger when the default moderation state is 'draft'.
    */
   public function testResourcePurgeDraft() {
-    $this->markTestSkipped('Flaky test fails inconsistently, typically line 119.');
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+    /** @var \Drupal\metastore_search\Search $metastore_search_service */
+    $metastore_search_service = \Drupal::service('dkan.metastore_search.service');
+
     $id_1 = uniqid(__FUNCTION__ . '1');
     $id_2 = uniqid(__FUNCTION__ . '2');
     $id_3 = uniqid(__FUNCTION__ . '3');
@@ -103,7 +108,7 @@ public function testResourcePurgeDraft() {
     // Post, update and publish a dataset with multiple, changing resources.
     $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv']);
     $this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
-    $this->getMetastore()->publish('dataset', $id_1);
+    $metastore_service->publish('dataset', $id_1);
     $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
 
     /** @var \Drupal\common\DatasetInfo $datasetInfo */
@@ -122,13 +127,13 @@ public function testResourcePurgeDraft() {
     // Add more datasets, only publishing some.
     $this->storeDatasetRunQueues($id_2, '2.1', []);
     $this->storeDatasetRunQueues($id_3, '3.1', []);
-    $this->getMetastore()->publish('dataset', $id_2);
+    $metastore_service->publish('dataset', $id_2);
     // Reindex.
     $index = Index::load('dkan');
     $index->clear();
     $index->indexItems();
     // Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
-    $searchResults = $this->getMetastoreSearch()->search();
+    $searchResults = $metastore_search_service->search();
     $this->assertEquals(2, $searchResults->total);
     $this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
     $this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
@@ -206,7 +211,7 @@ public function testDeleteDistribution() {
     $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv']);
 
     // Get distribution id.
-    $dataset = $this->getMetastore()->get('dataset', $id_1);
+    $dataset = \Drupal::service('dkan.metastore.service')->get('dataset', $id_1);
     $datasetMetadata = $dataset->{'$'};
     $distributionId = $datasetMetadata["%Ref:distribution"][0]["identifier"];
 
@@ -227,6 +232,9 @@ public function testDeleteDistribution() {
    * Test local resource removal on datastore import.
    */
   public function testDatastoreImportDeleteLocalResource() {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+
     $id_1 = uniqid(__FUNCTION__ . '1');
     $id_2 = uniqid(__FUNCTION__ . '2');
 
@@ -241,7 +249,7 @@ public function testDatastoreImportDeleteLocalResource() {
     $this->storeDatasetRunQueues($id_1, '1', ['1.csv']);
 
     // Get local resource folder name.
-    $dataset = $this->getMetastore()->get('dataset', $id_1);
+    $dataset = $metastore_service->get('dataset', $id_1);
     $datasetMetadata = $dataset->{'$'};
     $resourceId = explode('__', $datasetMetadata["%Ref:distribution"][0]["data"]["%Ref:downloadURL"][0]["identifier"]);
     $refUuid = $resourceId[0] . '_' . $resourceId[1];
@@ -257,7 +265,7 @@ public function testDatastoreImportDeleteLocalResource() {
     $this->storeDatasetRunQueues($id_2, '2', ['2.csv']);
 
     // Get local resource folder name.
-    $dataset = $this->getMetastore()->get('dataset', $id_2);
+    $dataset = $metastore_service->get('dataset', $id_2);
     $datasetMetadata = $dataset->{'$'};
     $resourceId = explode('__', $datasetMetadata["%Ref:distribution"][0]["data"]["%Ref:downloadURL"][0]["identifier"]);
     $refUuid = $resourceId[0] . '_' . $resourceId[1];
@@ -270,17 +278,20 @@ public function testDatastoreImportDeleteLocalResource() {
   }
 
   private function datasetPostAndRetrieve(): object {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+
     $datasetRootedJsonData = $this->getData(123, 'Test #1', ['district_centerpoints_small.csv']);
     $dataset = json_decode($datasetRootedJsonData);
 
-    $uuid = $this->getMetastore()->post('dataset', $datasetRootedJsonData);
+    $uuid = $metastore_service->post('dataset', $datasetRootedJsonData);
 
     $this->assertEquals(
       $dataset->identifier,
       $uuid
     );
 
-    $datasetRootedJsonData = $this->getMetastore()->get('dataset', $uuid);
+    $datasetRootedJsonData = $metastore_service->get('dataset', $uuid);
     $this->assertIsString("$datasetRootedJsonData");
 
     $retrievedDataset = json_decode($datasetRootedJsonData);
@@ -461,14 +472,16 @@ private function queryResource(object $resource, string $queryString) {
   }
 
   private function httpVerbHandler(string $method, RootedJsonData $json, $dataset) {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
 
     if ($method == 'post') {
-      $identifier = $this->getMetastore()->post('dataset', $json);
+      $identifier = $metastore_service->post('dataset', $json);
     }
     // PUT for now, refactor later if more verbs are needed.
     else {
       $id = $dataset->identifier;
-      $info = $this->getMetastore()->put('dataset', $id, $json);
+      $info = $metastore_service->put('dataset', $id, $json);
       $identifier = $info['identifier'];
     }
 
@@ -495,18 +508,4 @@ private function getNodeStorage(): NodeStorage {
     return \Drupal::service('entity_type.manager')->getStorage('node');
   }
 
-  /**
-   * @return \Drupal\metastore_search\Search
-   */
-  private function getMetastoreSearch() : Search {
-    return \Drupal::service('dkan.metastore_search.service');
-  }
-
-  /**
-   * @return \Drupal\metastore\Service
-   */
-  private function getMetastore(): Metastore {
-    return \Drupal::service('dkan.metastore.service');
-  }
-
 }

From 83542cd433921166d9a8db7645e3debe8c94c51c Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Thu, 20 Apr 2023 09:17:47 -0500
Subject: [PATCH 02/23] undo coversnothing

---
 .../datastore/tests/src/Functional/DictionaryEnforcerTest.php  | 3 +--
 .../tests/src/Functional/Service/ResourcePurgerTest.php        | 1 -
 .../Functional/Plugin/Action/HideCurrentRevisionActionTest.php | 3 +--
 .../metastore/tests/src/Functional/Api1/DatasetItemTest.php    | 3 ---
 .../tests/src/Functional/MetastoreApiPageCacheTest.php         | 1 -
 tests/src/Functional/DatasetTest.php                           | 3 +--
 6 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 7e9b080a94..8c8c2c9192 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,8 +18,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @coversNothing
- */
+  */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
   use GetDataTrait, CleanUp;
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index 8768a95081..667f552440 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,7 +13,6 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @coversNothing
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
diff --git a/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php b/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
index 84bf80cbf5..92b398336e 100644
--- a/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
+++ b/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
@@ -18,8 +18,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @coversNothing
- */
+  */
 class HideCurrentRevisionActionTest extends ExistingSiteBase {
   use CleanUp;
   use UserCreationTrait;
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
index 1c773ffbd4..f15c860140 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
@@ -5,9 +5,6 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 use GuzzleHttp\RequestOptions;
 
-/**
- * @coversNothing
- */
 class DatasetItemTest extends Api1TestBase {
 
   public function getEndpoint():string {
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index f6ba614758..df5ecaaa16 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -15,7 +15,6 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @coversNothing
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index af7663fab5..bcde546d56 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -22,8 +22,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @coversNothing
- */
+  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;
 

From a12e4cb840f05252dc5e8cb93488372bc76c27b3 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Thu, 20 Apr 2023 09:34:50 -0500
Subject: [PATCH 03/23] some reverts...

---
 .../datastore/tests/src/Functional/DictionaryEnforcerTest.php   | 2 +-
 .../tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php         | 1 -
 .../Functional/Plugin/Action/HideCurrentRevisionActionTest.php  | 2 +-
 tests/src/Functional/DatasetTest.php                            | 2 +-
 4 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 8c8c2c9192..9b7b13ba61 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,7 +18,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
-  */
+ */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
   use GetDataTrait, CleanUp;
diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index 07a43d95fc..e5207f3dc4 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -174,7 +174,6 @@ public function testSerialization() {
    *
    */
   public function testMultiplePasses() {
-    $this->markTestSkipped('flaky test');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();
diff --git a/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php b/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
index 92b398336e..7c536d2878 100644
--- a/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
+++ b/modules/metastore/modules/metastore_admin/tests/src/Functional/Plugin/Action/HideCurrentRevisionActionTest.php
@@ -18,7 +18,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
-  */
+ */
 class HideCurrentRevisionActionTest extends ExistingSiteBase {
   use CleanUp;
   use UserCreationTrait;
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index bcde546d56..4d42bb3204 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -22,7 +22,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
-  */
+ */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;
 

From b7f2c1251b55132d5e4897f20dc6ac3c42178c1c Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Thu, 20 Apr 2023 11:17:36 -0500
Subject: [PATCH 04/23] some

---
 modules/metastore/tests/src/Unit/ServiceTest.php | 6 +++---
 tests/src/Functional/DatasetTest.php             | 9 ++++-----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/modules/metastore/tests/src/Unit/ServiceTest.php b/modules/metastore/tests/src/Unit/ServiceTest.php
index f9befd9d25..794ffed1b6 100644
--- a/modules/metastore/tests/src/Unit/ServiceTest.php
+++ b/modules/metastore/tests/src/Unit/ServiceTest.php
@@ -397,9 +397,9 @@ public function testGetCatalog() {
   }
 
   /**
-   * @return \Drupal\common\Tests\Mock\Chain
+   * @return \MockChain\Chain
    */
-  public static function getCommonMockChain(TestCase $case, Options $services = null) {
+  public static function getCommonMockChain(TestCase $case, Options $services = null): Chain {
 
     $options = (new Options)
       ->add('dkan.metastore.schema_retriever', SchemaRetriever::class)
@@ -416,7 +416,7 @@ public static function getCommonMockChain(TestCase $case, Options $services = nu
       ->add(SchemaRetriever::class, "retrieve", json_encode(['foo' => 'bar']));
   }
 
-  public static function getValidMetadataFactory(TestCase $case) {
+  public static function getValidMetadataFactory(TestCase $case): ValidMetadataFactory {
     $options = (new Options())
       ->add('metastore.schema_retriever', SchemaRetriever::class)
       ->index(0);
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 4d42bb3204..f84d311f89 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -7,8 +7,7 @@
 use Drupal\datastore\Service\ResourceLocalizer;
 use Drupal\harvest\Load\Dataset;
 use Drupal\harvest\Service as Harvester;
-use Drupal\metastore\Service as Metastore;
-use Drupal\metastore_search\Search;
+use Drupal\metastore\ValidMetadataFactory;
 use Drupal\node\NodeStorage;
 use Drupal\search_api\Entity\Index;
 use Drupal\Tests\common\Traits\CleanUp;
@@ -29,7 +28,7 @@ class DatasetTest extends ExistingSiteBase {
   private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
   private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';
 
-  private $validMetadataFactory;
+  private ValidMetadataFactory $validMetadataFactory;
 
   public function setUp(): void {
     parent::setUp();
@@ -321,7 +320,7 @@ private function changeDatasetsResourceOutputPerspective(string $perspective = D
   }
 
   private function getResourceDatastoreTable(object $resource) {
-    return "{$resource->identifier}__{$resource->version}";
+    return "{$resource->identifier}__$resource->version";
   }
 
   private function getResourceFromDataset(object $dataset) {
@@ -372,7 +371,7 @@ private function getData(string $identifier, string $title, array $downloadUrls)
 
     foreach ($downloadUrls as $key => $downloadUrl) {
       $distribution = new \stdClass();
-      $distribution->title = "Distribution #{$key} for {$identifier}";
+      $distribution->title = "Distribution #$key for $identifier";
       $distribution->downloadURL = $this->getDownloadUrl($downloadUrl);
       $distribution->mediaType = "text/csv";
 

From 3fdde8038b49b8760517ae8e74f52bca731cf47e Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Thu, 20 Apr 2023 11:29:27 -0500
Subject: [PATCH 05/23] codeclimate

---
 modules/datastore/src/SqlEndpoint/Service.php | 2 --
 modules/harvest/src/Load/Dataset.php          | 8 ++++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/modules/datastore/src/SqlEndpoint/Service.php b/modules/datastore/src/SqlEndpoint/Service.php
index 9cd4027321..2d03f2d7d9 100644
--- a/modules/datastore/src/SqlEndpoint/Service.php
+++ b/modules/datastore/src/SqlEndpoint/Service.php
@@ -4,7 +4,6 @@
 
 use Drupal\common\DataResource;
 use Drupal\Core\Config\ConfigFactory;
-use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
 use Drupal\common\Storage\Query;
 use Drupal\datastore\Service as DatastoreService;
 use Drupal\datastore\SqlEndpoint\Helper\GetStringsFromStateMachineExecution;
@@ -12,7 +11,6 @@
 use Maquina\StateMachine\Machine;
 use Maquina\StateMachine\MachineOfMachines;
 use SqlParser\SqlParser;
-use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * SQL endpoint service.
diff --git a/modules/harvest/src/Load/Dataset.php b/modules/harvest/src/Load/Dataset.php
index 7c5e736c76..0ec5d4d4a4 100644
--- a/modules/harvest/src/Load/Dataset.php
+++ b/modules/harvest/src/Load/Dataset.php
@@ -11,8 +11,16 @@
  */
 class Dataset extends Load {
 
+  /**
+   * Metastore service.
+   *
+   * @var \Drupal\metastore\Service
+   */
   protected Service $metastoreService;
 
+  /**
+   * {@inheritDoc}
+   */
   public function __construct($harvest_plan, $hash_storage, $item_storage) {
     $this->metastoreService = \Drupal::service('dkan.metastore.service');
     parent::__construct($harvest_plan, $hash_storage, $item_storage);

From baca47cb44db406030491426f2bfa82d483b0e0c Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Thu, 20 Apr 2023 12:53:49 -0500
Subject: [PATCH 06/23] find public with service

---
 tests/src/Functional/DatasetTest.php | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index f84d311f89..46f3cc047e 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -109,9 +109,7 @@ public function testResourcePurgeDraft() {
     $metastore_service->publish('dataset', $id_1);
     $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
 
-    /** @var \Drupal\common\DatasetInfo $datasetInfo */
-    $datasetInfo = \Drupal::service('dkan.common.dataset_info');
-    $info = $datasetInfo->gather($id_1);
+    $info = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
     $this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
     $this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
     $this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
@@ -445,11 +443,14 @@ private function countTables() {
     return count($tables);
   }
 
+  /**
+   * @return array
+   *   File names.
+   */
   private function checkFiles() {
     /** @var \Drupal\Core\File\FileSystemInterface $fileSystem */
     $fileSystem = \Drupal::service('file_system');
-
-    $dir = DRUPAL_ROOT . "/sites/default/files/resources";
+    $dir = $fileSystem->realpath('public://resources');
     // Nothing to check if the resource folder does not exist.
     if (!is_dir($dir)) {
       return [];

From 054826a9499cc5cb5d02c13c3e81091953755c85 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Thu, 20 Apr 2023 14:20:33 -0500
Subject: [PATCH 07/23] new file for resource purge test

---
 .../DatasetResourcePurgeDraftTest.php         | 343 ++++++++++++++++++
 tests/src/Functional/DatasetTest.php          |  50 ---
 2 files changed, 343 insertions(+), 50 deletions(-)
 create mode 100644 tests/src/Functional/DatasetResourcePurgeDraftTest.php

diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTest.php b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
new file mode 100644
index 0000000000..6e3ba0629c
--- /dev/null
+++ b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
@@ -0,0 +1,343 @@
+<?php
+
+namespace Drupal\Tests\dkan\Functional;
+
+use Drupal\common\DataResource;
+use Drupal\Core\Queue\QueueFactory;
+use Drupal\datastore\Service\ResourceLocalizer;
+use Drupal\harvest\Load\Dataset;
+use Drupal\harvest\Service as Harvester;
+use Drupal\metastore\ValidMetadataFactory;
+use Drupal\node\NodeStorage;
+use Drupal\search_api\Entity\Index;
+use Drupal\Tests\common\Traits\CleanUp;
+use Drupal\Tests\metastore\Unit\ServiceTest;
+use Harvest\ETL\Extract\DataJson;
+use RootedData\RootedJsonData;
+use weitzman\DrupalTestTraits\ExistingSiteBase;
+
+/**
+ * Class DatasetTest
+ *
+ * @package Drupal\Tests\dkan\Functional
+ * @group dkan
+ */
+class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
+  use CleanUp;
+
+  private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
+  private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';
+
+  private ValidMetadataFactory $validMetadataFactory;
+
+  public function setUp(): void {
+    parent::setUp();
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
+    $this->setDefaultModerationState();
+    $this->changeDatasetsResourceOutputPerspective();
+    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
+  }
+
+  public function tearDown(): void {
+    parent::tearDown();
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
+    $this->setDefaultModerationState();
+    $this->changeDatasetsResourceOutputPerspective();
+  }
+
+  /**
+   * Test the resource purger when the default moderation state is 'draft'.
+   */
+  public function testResourcePurgeDraft() {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+    /** @var \Drupal\metastore_search\Search $metastore_search_service */
+    $metastore_search_service = \Drupal::service('dkan.metastore_search.service');
+
+    $id_1 = uniqid(__FUNCTION__ . '1');
+    $id_2 = uniqid(__FUNCTION__ . '2');
+    $id_3 = uniqid(__FUNCTION__ . '3');
+
+    $this->setDefaultModerationState('draft');
+
+    // Post, update and publish a dataset with multiple, changing resources.
+    $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv']);
+    $this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
+    $metastore_service->publish('dataset', $id_1);
+    $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
+
+    $info = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
+    $this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
+    $this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
+    $this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
+    $this->assertStringEndsWith('1.csv', $info['published_revision']['distributions'][1]['file_path']);
+
+    // Verify that only the resources associated with the published and the
+    // latest revision.
+    $this->assertEquals(['1.csv', '3.csv', '5.csv'], $this->checkFiles());
+    $this->assertEquals(3, $this->countTables());
+
+    // Add more datasets, only publishing some.
+    $this->storeDatasetRunQueues($id_2, '2.1', []);
+    $this->storeDatasetRunQueues($id_3, '3.1', []);
+    $metastore_service->publish('dataset', $id_2);
+    // Reindex.
+    $index = Index::load('dkan');
+    $index->clear();
+    $index->indexItems();
+    // Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
+    $searchResults = $metastore_search_service->search();
+    $this->assertEquals(2, $searchResults->total);
+    $this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
+    $this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
+    $this->assertArrayHasKey('dkan_dataset/' . $id_2, $searchResults->results);
+    $this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
+  }
+
+  private function datasetPostAndRetrieve(): object {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+
+    $datasetRootedJsonData = $this->getData(123, 'Test #1', ['district_centerpoints_small.csv']);
+    $dataset = json_decode($datasetRootedJsonData);
+
+    $uuid = $metastore_service->post('dataset', $datasetRootedJsonData);
+
+    $this->assertEquals(
+      $dataset->identifier,
+      $uuid
+    );
+
+    $datasetRootedJsonData = $metastore_service->get('dataset', $uuid);
+    $this->assertIsString("$datasetRootedJsonData");
+
+    $retrievedDataset = json_decode($datasetRootedJsonData);
+
+    $this->assertEquals(
+      $retrievedDataset->identifier,
+      $uuid
+    );
+
+    return $retrievedDataset;
+  }
+
+  private function datastoreImportAndQuery() {
+    $dataset = $this->datasetPostAndRetrieve();
+    $resource = $this->getResourceFromDataset($dataset);
+
+    $this->runQueues(['datastore_import']);
+
+    $queryString = "[SELECT * FROM {$this->getResourceDatastoreTable($resource)}][WHERE lon = \"61.33\"][ORDER BY lat DESC][LIMIT 1 OFFSET 0];";
+    $this->queryResource($resource, $queryString);
+
+    /**/
+  }
+
+  private function changeDatasetsResourceOutputPerspective(string $perspective = DataResource::DEFAULT_SOURCE_PERSPECTIVE) {
+    $display = &drupal_static('metastore_resource_mapper_display');
+    $display = $perspective;
+  }
+
+  private function getResourceDatastoreTable(object $resource) {
+    return "{$resource->identifier}__$resource->version";
+  }
+
+  private function getResourceFromDataset(object $dataset) {
+    $this->assertTrue(isset($dataset->{"%Ref:distribution"}));
+    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]));
+    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data));
+    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}));
+    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}[0]));
+    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}[0]->data));
+
+    return $dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}[0]->data;
+  }
+
+  private function getDownloadUrl(string $filename) {
+    return self::S3_PREFIX . '/' . $filename;
+  }
+
+  /**
+   * Generate dataset metadata, possibly with multiple distributions.
+   *
+   * @param string $identifier
+   *   Dataset identifier.
+   * @param string $title
+   *   Dataset title.
+   * @param array $downloadUrls
+   *   Array of resource files URLs for this dataset.
+   *
+   * @return \RootedData\RootedJsonData
+   *   Json encoded string of this dataset's metadata, or FALSE if error.
+   */
+  private function getData(string $identifier, string $title, array $downloadUrls): RootedJsonData {
+
+    $data = new \stdClass();
+    $data->title = $title;
+    $data->description = "Some description.";
+    $data->identifier = $identifier;
+    $data->accessLevel = "public";
+    $data->modified = "06-04-2020";
+    $data->keyword = ["some keyword"];
+    $data->distribution = [];
+    $data->publisher = (object) [
+      'name' => 'Test Publisher',
+    ];
+    $data->contactPoint = (object) [
+      'fn' => 'Test Name',
+      'hasEmail' => 'test@example.com',
+    ];
+
+    foreach ($downloadUrls as $key => $downloadUrl) {
+      $distribution = new \stdClass();
+      $distribution->title = "Distribution #$key for $identifier";
+      $distribution->downloadURL = $this->getDownloadUrl($downloadUrl);
+      $distribution->mediaType = "text/csv";
+
+      $data->distribution[] = $distribution;
+    }
+
+    return $this->validMetadataFactory->get(json_encode($data), 'dataset');
+  }
+
+  /**
+   * Generate a harvest plan object.
+   */
+  private function getPlan(string $identifier, string $testFilename) : \stdClass {
+    return (object) [
+      'identifier' => $identifier,
+      'extract' => (object) [
+        'type' => DataJson::class,
+        'uri' => 'file://' . __DIR__ . '/../../files/' . $testFilename,
+      ],
+      'transforms' => [],
+      'load' => (object) [
+        'type' => Dataset::class,
+      ],
+    ];
+  }
+
+  /**
+   * Get a dataset's moderation state.
+   */
+  private function getModerationState(string $uuid) : string {
+    $nodeStorage = $this->getNodeStorage();
+    $datasets = $nodeStorage->loadByProperties(['uuid' => $uuid]);
+    if (FALSE !== ($dataset = reset($datasets))) {
+      return $dataset->get('moderation_state')->getString();
+    }
+    return '';
+  }
+
+  /**
+   * Store or update a dataset,run datastore_import and resource_purger queues.
+   */
+  private function storeDatasetRunQueues(string $identifier, string $title, array $filenames, string $method = 'post') {
+    $datasetRootedJsonData = $this->getData($identifier, $title, $filenames);
+    $this->httpVerbHandler($method, $datasetRootedJsonData, json_decode($datasetRootedJsonData));
+
+    // Simulate a cron on queues relevant to this scenario.
+    $this->runQueues(['datastore_import', 'resource_purger']);
+  }
+
+  /**
+   * Process queues in a predictable order.
+   */
+  private function runQueues(array $relevantQueues = []) {
+    /** @var \Drupal\Core\Queue\QueueWorkerManager $queueWorkerManager */
+    $queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
+    foreach ($relevantQueues as $queueName) {
+      $worker = $queueWorkerManager->createInstance($queueName);
+      $queue = $this->getQueueService()->get($queueName);
+      while ($item = $queue->claimItem()) {
+        $worker->processItem($item->data);
+        $queue->deleteItem($item);
+      }
+    }
+  }
+
+  private function countTables() {
+    /** @var $db \Drupal\Core\Database\Connection */
+    $db = \Drupal::service('database');
+
+    $tables = $db->schema()->findTables("datastore_%");
+    return count($tables);
+  }
+
+  /**
+   * @return array
+   *   File names.
+   */
+  private function checkFiles() {
+    /** @var \Drupal\Core\File\FileSystemInterface $fileSystem */
+    $fileSystem = \Drupal::service('file_system');
+    $dir = $fileSystem->realpath('public://resources');
+    // Nothing to check if the resource folder does not exist.
+    if (!is_dir($dir)) {
+      return [];
+    }
+    $filesObjects = $fileSystem->scanDirectory($dir, "/.*\.csv$/i", ['recurse' => TRUE]);
+    $filenames = array_values(array_map(function ($obj) {
+      return str_replace(self::FILENAME_PREFIX, '', $obj->filename);
+    }, $filesObjects));
+    sort($filenames);
+    return $filenames;
+  }
+
+  private function queryResource(object $resource, string $queryString) {
+    /** @var $sqlEndpoint \Drupal\datastore\SqlEndpoint\Service */
+    $sqlEndpoint = \Drupal::service('dkan.datastore.sql_endpoint.service');
+    $results = $sqlEndpoint->runQuery($queryString);
+    $this->assertGreaterThan(0, count($results));
+  }
+
+  private function httpVerbHandler(string $method, RootedJsonData $json, $dataset) {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+
+    if ($method == 'post') {
+      $identifier = $metastore_service->post('dataset', $json);
+    }
+    // PUT for now, refactor later if more verbs are needed.
+    else {
+      $id = $dataset->identifier;
+      $info = $metastore_service->put('dataset', $id, $json);
+      $identifier = $info['identifier'];
+    }
+
+    return $identifier;
+  }
+
+  private function setDefaultModerationState($state = 'published') {
+    /** @var \Drupal\Core\Config\ConfigFactory $config */
+    $config = \Drupal::service('config.factory');
+    $defaultModerationState = $config->getEditable('workflows.workflow.dkan_publishing');
+    $defaultModerationState->set('type_settings.default_moderation_state', $state);
+    $defaultModerationState->save();
+  }
+
+  private function getQueueService() : QueueFactory {
+    return \Drupal::service('queue');
+  }
+
+  private function getHarvester() : Harvester {
+    return \Drupal::service('dkan.harvest.service');
+  }
+
+  private function getNodeStorage(): NodeStorage {
+    return \Drupal::service('entity_type.manager')->getStorage('node');
+  }
+
+}
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 46f3cc047e..db54956da5 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -88,56 +88,6 @@ public function testResourcePurgePublished() {
     $this->assertEquals(2, $this->countTables());
   }
 
-  /**
-   * Test the resource purger when the default moderation state is 'draft'.
-   */
-  public function testResourcePurgeDraft() {
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-    /** @var \Drupal\metastore_search\Search $metastore_search_service */
-    $metastore_search_service = \Drupal::service('dkan.metastore_search.service');
-
-    $id_1 = uniqid(__FUNCTION__ . '1');
-    $id_2 = uniqid(__FUNCTION__ . '2');
-    $id_3 = uniqid(__FUNCTION__ . '3');
-
-    $this->setDefaultModerationState('draft');
-
-    // Post, update and publish a dataset with multiple, changing resources.
-    $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv']);
-    $this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
-    $metastore_service->publish('dataset', $id_1);
-    $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
-
-    $info = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
-    $this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
-    $this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
-    $this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
-    $this->assertStringEndsWith('1.csv', $info['published_revision']['distributions'][1]['file_path']);
-
-    // Verify that only the resources associated with the published and the
-    // latest revision.
-    $this->assertEquals(['1.csv', '3.csv', '5.csv'], $this->checkFiles());
-    $this->assertEquals(3, $this->countTables());
-
-    // Add more datasets, only publishing some.
-    $this->storeDatasetRunQueues($id_2, '2.1', []);
-    $this->storeDatasetRunQueues($id_3, '3.1', []);
-    $metastore_service->publish('dataset', $id_2);
-    // Reindex.
-    $index = Index::load('dkan');
-    $index->clear();
-    $index->indexItems();
-    // Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
-    $searchResults = $metastore_search_service->search();
-    $this->assertEquals(2, $searchResults->total);
-    $this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
-    $this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
-    $this->assertArrayHasKey('dkan_dataset/' . $id_2, $searchResults->results);
-    $this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
-  }
-
-
   /**
    * Test archiving of datasets after a harvest
    */

From 3e5041b45d8b78d58addf323898baf47537e7189 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Fri, 21 Apr 2023 09:45:02 -0500
Subject: [PATCH 08/23] some test stuff

---
 .../DatasetResourcePurgeDraftTest.php          | 18 ++++++------------
 tests/src/Functional/DatasetTest.php           |  9 ---------
 2 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTest.php b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
index 6e3ba0629c..ad253a443a 100644
--- a/tests/src/Functional/DatasetResourcePurgeDraftTest.php
+++ b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
@@ -32,15 +32,6 @@ class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
-    $this->removeHarvests();
-    $this->removeAllNodes();
-    $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
-    $this->setDefaultModerationState();
-    $this->changeDatasetsResourceOutputPerspective();
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
@@ -246,10 +237,11 @@ private function getModerationState(string $uuid) : string {
    */
   private function storeDatasetRunQueues(string $identifier, string $title, array $filenames, string $method = 'post') {
     $datasetRootedJsonData = $this->getData($identifier, $title, $filenames);
-    $this->httpVerbHandler($method, $datasetRootedJsonData, json_decode($datasetRootedJsonData));
+    $identifier = $this->httpVerbHandler($method, $datasetRootedJsonData, json_decode($datasetRootedJsonData));
 
     // Simulate a cron on queues relevant to this scenario.
     $this->runQueues(['datastore_import', 'resource_purger']);
+    return $identifier;
   }
 
   /**
@@ -258,9 +250,11 @@ private function storeDatasetRunQueues(string $identifier, string $title, array
   private function runQueues(array $relevantQueues = []) {
     /** @var \Drupal\Core\Queue\QueueWorkerManager $queueWorkerManager */
     $queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
+    /** @var \Drupal\Core\Queue\QueueFactory $queue_factory */
+    $queue_factory = \Drupal::service('queue');
     foreach ($relevantQueues as $queueName) {
       $worker = $queueWorkerManager->createInstance($queueName);
-      $queue = $this->getQueueService()->get($queueName);
+      $queue = $queue_factory->get($queueName);
       while ($item = $queue->claimItem()) {
         $worker->processItem($item->data);
         $queue->deleteItem($item);
@@ -303,7 +297,7 @@ private function queryResource(object $resource, string $queryString) {
     $this->assertGreaterThan(0, count($results));
   }
 
-  private function httpVerbHandler(string $method, RootedJsonData $json, $dataset) {
+  private function httpVerbHandler(string $method, RootedJsonData $json, $dataset): string {
     /** @var \Drupal\metastore\Service $metastore_service */
     $metastore_service = \Drupal::service('dkan.metastore.service');
 
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index db54956da5..6f168d5ad7 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -32,15 +32,6 @@ class DatasetTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
-    $this->removeHarvests();
-    $this->removeAllNodes();
-    $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
-    $this->setDefaultModerationState();
-    $this->changeDatasetsResourceOutputPerspective();
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 

From d4ca28a63b1a21c42f7f89a36604efc9012b8d8b Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Mon, 24 Apr 2023 13:08:10 -0500
Subject: [PATCH 09/23] do not go

---
 ...erTest.php => DictionaryEnforcerTe_st.php} | 15 +++-
 .../Functional/Service/ResourcePurgerTest.php | 22 +++---
 .../Unit/Plugin/QueueWorker/ImportJobTest.php |  1 +
 .../DatasetResourcePurgeDraftTest.php         | 77 +------------------
 tests/src/Functional/DatasetTest.php          |  1 +
 5 files changed, 28 insertions(+), 88 deletions(-)
 rename modules/datastore/tests/src/Functional/{DictionaryEnforcerTest.php => DictionaryEnforcerTe_st.php} (92%)

diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTe_st.php
similarity index 92%
rename from modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
rename to modules/datastore/tests/src/Functional/DictionaryEnforcerTe_st.php
index 9b7b13ba61..d9487749c8 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTe_st.php
@@ -18,8 +18,9 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
+ * @group dataset
  */
-class DictionaryEnforcerTest extends ExistingSiteBase {
+class DictionaryEnforcerTe_st extends ExistingSiteBase {
 
   use GetDataTrait, CleanUp;
 
@@ -122,6 +123,11 @@ public function setUp(): void {
   public function tearDown(): void {
     parent::tearDown();
     $this->removeAllMappedFiles();
+    // Clean up our CSV upload.
+    /** @var \Drupal\Core\File\FileSystemInterface $file_system */
+    $file_system = \Drupal::service('file_system');
+    $upload_path = $file_system->realpath(self::UPLOAD_LOCATION);
+    $file_system->delete($upload_path . '/' . self::RESOURCE_FILE);
   }
 
   /**
@@ -189,7 +195,12 @@ public function testDictionaryEnforcement(): void {
 
     // Build dataset.
     $dataset_id = $this->uuid->generate();
-    $dataset = $this->validMetadataFactory->get($this->getDataset($dataset_id, 'Test ' . $dataset_id, [$this->resourceUrl], TRUE), 'dataset');
+    $dataset = $this->validMetadataFactory->get(
+      $this->getDataset($dataset_id, 'Test ' . $dataset_id, [$this->resourceUrl], TRUE),
+      'dataset'
+    );
+//    $this->assertTrue(TRUE);
+//    return;
     // Create dataset.
     $this->metastore->post('dataset', $dataset);
     $this->metastore->publish('dataset', $dataset_id);
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index 667f552440..5fdf02751f 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,6 +13,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
+ * @group dataset
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
@@ -70,16 +71,6 @@ class ResourcePurgerTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
-
-    // Prepare environment.
-    $this->removeHarvests();
-    $this->removeAllNodes();
-    $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
-
     // Initialize services.
     $this->datasetStorage = \Drupal::service('dkan.metastore.storage')->getInstance('dataset');
     $this->datastore = \Drupal::service('dkan.datastore.service');
@@ -90,6 +81,17 @@ public function setUp(): void {
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
+  protected function tearDown(): void {
+    parent::tearDown();
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
+  }
+
   /**
    * Test deleting a dataset doesn't delete other datasets sharing a resource.
    */
diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index e5207f3dc4..5c242913de 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -174,6 +174,7 @@ public function testSerialization() {
    *
    */
   public function testMultiplePasses() {
+    $this->markTestSkipped('foo');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();
diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTest.php b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
index ad253a443a..10074e7700 100644
--- a/tests/src/Functional/DatasetResourcePurgeDraftTest.php
+++ b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
@@ -21,6 +21,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @group dataset
  */
 class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
   use CleanUp;
@@ -97,45 +98,6 @@ public function testResourcePurgeDraft() {
     $this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
   }
 
-  private function datasetPostAndRetrieve(): object {
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-
-    $datasetRootedJsonData = $this->getData(123, 'Test #1', ['district_centerpoints_small.csv']);
-    $dataset = json_decode($datasetRootedJsonData);
-
-    $uuid = $metastore_service->post('dataset', $datasetRootedJsonData);
-
-    $this->assertEquals(
-      $dataset->identifier,
-      $uuid
-    );
-
-    $datasetRootedJsonData = $metastore_service->get('dataset', $uuid);
-    $this->assertIsString("$datasetRootedJsonData");
-
-    $retrievedDataset = json_decode($datasetRootedJsonData);
-
-    $this->assertEquals(
-      $retrievedDataset->identifier,
-      $uuid
-    );
-
-    return $retrievedDataset;
-  }
-
-  private function datastoreImportAndQuery() {
-    $dataset = $this->datasetPostAndRetrieve();
-    $resource = $this->getResourceFromDataset($dataset);
-
-    $this->runQueues(['datastore_import']);
-
-    $queryString = "[SELECT * FROM {$this->getResourceDatastoreTable($resource)}][WHERE lon = \"61.33\"][ORDER BY lat DESC][LIMIT 1 OFFSET 0];";
-    $this->queryResource($resource, $queryString);
-
-    /**/
-  }
-
   private function changeDatasetsResourceOutputPerspective(string $perspective = DataResource::DEFAULT_SOURCE_PERSPECTIVE) {
     $display = &drupal_static('metastore_resource_mapper_display');
     $display = $perspective;
@@ -203,35 +165,6 @@ private function getData(string $identifier, string $title, array $downloadUrls)
     return $this->validMetadataFactory->get(json_encode($data), 'dataset');
   }
 
-  /**
-   * Generate a harvest plan object.
-   */
-  private function getPlan(string $identifier, string $testFilename) : \stdClass {
-    return (object) [
-      'identifier' => $identifier,
-      'extract' => (object) [
-        'type' => DataJson::class,
-        'uri' => 'file://' . __DIR__ . '/../../files/' . $testFilename,
-      ],
-      'transforms' => [],
-      'load' => (object) [
-        'type' => Dataset::class,
-      ],
-    ];
-  }
-
-  /**
-   * Get a dataset's moderation state.
-   */
-  private function getModerationState(string $uuid) : string {
-    $nodeStorage = $this->getNodeStorage();
-    $datasets = $nodeStorage->loadByProperties(['uuid' => $uuid]);
-    if (FALSE !== ($dataset = reset($datasets))) {
-      return $dataset->get('moderation_state')->getString();
-    }
-    return '';
-  }
-
   /**
    * Store or update a dataset,run datastore_import and resource_purger queues.
    */
@@ -322,14 +255,6 @@ private function setDefaultModerationState($state = 'published') {
     $defaultModerationState->save();
   }
 
-  private function getQueueService() : QueueFactory {
-    return \Drupal::service('queue');
-  }
-
-  private function getHarvester() : Harvester {
-    return \Drupal::service('dkan.harvest.service');
-  }
-
   private function getNodeStorage(): NodeStorage {
     return \Drupal::service('entity_type.manager')->getStorage('node');
   }
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 6f168d5ad7..f93c8f52d7 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -21,6 +21,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @group dataset
  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;

From a4962be840add93429f85edb366c940c9bfcf1e6 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Tue, 25 Apr 2023 09:29:58 -0500
Subject: [PATCH 10/23] run only dataset in ci

---
 .circleci/config.yml                          |  2 ++
 ...erTe_st.php => DictionaryEnforcerTest.php} |  2 +-
 .../src/Functional/OnPreReferenceTest.php     | 16 +++++-----
 .../src/Functional/OrphanCheckerTest.php      | 29 ++++++++++++-------
 .../DatasetResourcePurgeDraftTest.php         |  2 +-
 tests/src/Functional/DatasetTest.php          | 18 ++++++++----
 6 files changed, 41 insertions(+), 28 deletions(-)
 rename modules/datastore/tests/src/Functional/{DictionaryEnforcerTe_st.php => DictionaryEnforcerTest.php} (99%)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3354dc34e6..fce3a9d7af 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -153,6 +153,7 @@ jobs:
                   ddev xdebug on
                   $CIRCLE_WORKING_DIRECTORY/cc-test-reporter before-build
                   ddev dkan-phpunit \
+                    --group dataset \
                     --coverage-clover /var/www/html/docroot/modules/contrib/dkan/clover.xml \
                     --coverage-html /var/www/html/docroot/modules/contrib/dkan/coverage-html \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
@@ -177,6 +178,7 @@ jobs:
                 name: Run PHPUnit tests
                 command: |
                   ddev dkan-phpunit \
+                    --group dataset \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
       - store_test_results:
           path: dkan/junit
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTe_st.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
similarity index 99%
rename from modules/datastore/tests/src/Functional/DictionaryEnforcerTe_st.php
rename to modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index d9487749c8..0fd24b75df 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTe_st.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -20,7 +20,7 @@
  * @group datastore
  * @group dataset
  */
-class DictionaryEnforcerTe_st extends ExistingSiteBase {
+class DictionaryEnforcerTest extends ExistingSiteBase {
 
   use GetDataTrait, CleanUp;
 
diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index 8c88db05b3..90368d56f8 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -6,7 +6,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- *
+ * @group dataset
  */
 class OnPreReferenceTest extends ExistingSiteBase {
   use CleanUp;
@@ -16,12 +16,12 @@ class OnPreReferenceTest extends ExistingSiteBase {
   /**
    *
    */
-  private function getData($downloadUrl) {
+  private function getData($downloadUrl, $id) {
     return '
     {
       "title": "Test #1",
       "description": "Yep",
-      "identifier": "123",
+      "identifier": "' . $id . '",
       "accessLevel": "public",
       "modified": "06-04-2020",
       "keyword": ["hello"],
@@ -47,7 +47,8 @@ public function test() {
     $datastore_settings->save();
 
     // Test posting a dataset to the metastore.
-    $data = $this->getData($this->downloadUrl);
+    $id = uniqid();
+    $data = $this->getData($this->downloadUrl, $id);
     /** @var \Drupal\metastore\Service $metastore */
     $metastore = \Drupal::service('dkan.metastore.service');
     $dataset = $metastore->getValidMetadataFactory()->get($data, 'dataset');
@@ -58,16 +59,13 @@ public function test() {
     $edited = json_encode($decoded);
 
     $dataset = $metastore->getValidMetadataFactory()->get($edited, 'dataset');
-    $metastore->patch('dataset', '123', $dataset);
+    $metastore->patch('dataset', $id, $dataset);
 
     $rev = drupal_static('metastore_resource_mapper_new_revision');
     $this->assertEquals(1, $rev);
   }
 
-  /**
-   *
-   */
-  public function tearDown(): void {
+  protected function tearDown(): void {
     parent::tearDown();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
diff --git a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
index cf727fd754..27393d6c43 100644
--- a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
+++ b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
@@ -13,6 +13,7 @@
  *
  * @package Drupal\Tests\metastore\Functional
  * @group metastore
+ * @group dataset
  */
 class OrphanCheckerTest extends ExistingSiteBase {
   use GetDataTrait;
@@ -27,6 +28,11 @@ class OrphanCheckerTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
+    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
+  }
+
+  protected function tearDown(): void {
+    parent::tearDown();
     $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
@@ -34,28 +40,32 @@ public function setUp(): void {
     $this->flushQueues();
     $this->removeFiles();
     $this->removeDatastoreTables();
-    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
   public function test() {
+    $id_1 = uniqid();
+    $id_2 = uniqid();
     /** @var $service \Drupal\metastore\Service */
     $service = \Drupal::service('dkan.metastore.service');
-    $dataset = $this->validMetadataFactory->get($this->getDataset(123, 'Test #1', ['district_centerpoints_small.csv']), 'dataset');
-    $service->post('dataset', $dataset);
-    $dataset2 = $this->validMetadataFactory->get($this->getDataset(456, 'Test #2', ['district_centerpoints_small.csv']), 'dataset');
+    $dataset = $this->validMetadataFactory->get($this->getDataset($id_1, 'Test #1', ['district_centerpoints_small.csv']), 'dataset');
+    $this->assertNotEmpty(
+      $service->post('dataset', $dataset)
+    );
+    $dataset2 = $this->validMetadataFactory->get($this->getDataset($id_2, 'Test #2', ['district_centerpoints_small.csv']), 'dataset');
     $service->post('dataset', $dataset2);
     $this->runQueues(['datastore_import']);
-    $service->delete('dataset', 123);
-    $success = $this->runQueues(['orphan_reference_processor']);
-    $this->assertNull($success);
+    $service->delete('dataset', $id_1);
+    $this->runQueues(['orphan_reference_processor']);
   }
 
   private function runQueues(array $relevantQueues = []) {
     /** @var \Drupal\Core\Queue\QueueWorkerManager $queueWorkerManager */
     $queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
+    /** @var QueueFactory $queue_factory */
+    $queue_factory = \Drupal::service('queue');
     foreach ($relevantQueues as $queueName) {
       $worker = $queueWorkerManager->createInstance($queueName);
-      $queue = $this->getQueueService()->get($queueName);
+      $queue = $queue_factory->get($queueName);
       while ($item = $queue->claimItem()) {
         $worker->processItem($item->data);
         $queue->deleteItem($item);
@@ -63,7 +73,4 @@ private function runQueues(array $relevantQueues = []) {
     }
   }
 
-  private function getQueueService() : QueueFactory {
-    return \Drupal::service('queue');
-  }
 }
diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTest.php b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
index 10074e7700..e79ece87d4 100644
--- a/tests/src/Functional/DatasetResourcePurgeDraftTest.php
+++ b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
@@ -21,7 +21,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group dataset
+ * @group _dataset
  */
 class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index f93c8f52d7..8f06c7e5d3 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -50,13 +50,15 @@ public function tearDown(): void {
   }
 
   public function testChangingDatasetResourcePerspectiveOnOutput() {
-    $this->datastoreImportAndQuery();
+    $this->markTestIncomplete('fails due to lack of schema');
+    $dataset_id = uniqid();
+    $this->datastoreImportAndQuery($dataset_id);
 
     drupal_flush_all_caches();
 
     $this->changeDatasetsResourceOutputPerspective(ResourceLocalizer::LOCAL_URL_PERSPECTIVE);
 
-    $metadata = \Drupal::service('dkan.metastore.service')->get('dataset', 123);
+    $metadata = \Drupal::service('dkan.metastore.service')->get('dataset', $dataset_id);
     $dataset = json_decode($metadata);
 
     $this->assertNotEquals(
@@ -215,11 +217,15 @@ public function testDatastoreImportDeleteLocalResource() {
     $datastoreSettings->set('delete_local_resource', $deleteLocalResourceOriginal)->save();
   }
 
-  private function datasetPostAndRetrieve(): object {
+  private function datasetPostAndRetrieve($dataset_id): object {
     /** @var \Drupal\metastore\Service $metastore_service */
     $metastore_service = \Drupal::service('dkan.metastore.service');
 
-    $datasetRootedJsonData = $this->getData(123, 'Test #1', ['district_centerpoints_small.csv']);
+    $datasetRootedJsonData = $this->getData(
+      $dataset_id,
+      'Test #1',
+      ['district_centerpoints_small.csv']
+    );
     $dataset = json_decode($datasetRootedJsonData);
 
     $uuid = $metastore_service->post('dataset', $datasetRootedJsonData);
@@ -242,8 +248,8 @@ private function datasetPostAndRetrieve(): object {
     return $retrievedDataset;
   }
 
-  private function datastoreImportAndQuery() {
-    $dataset = $this->datasetPostAndRetrieve();
+  private function datastoreImportAndQuery($dataset_id) {
+    $dataset = $this->datasetPostAndRetrieve($dataset_id);
     $resource = $this->getResourceFromDataset($dataset);
 
     $this->runQueues(['datastore_import']);

From 634ab4f648559e28c82d130fef18f9b3ba3df535 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Tue, 25 Apr 2023 09:48:37 -0500
Subject: [PATCH 11/23] more in the group

---
 ...ePurgeDraftTest.php => DatasetResourcePurgeDraftTe_st.php} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename tests/src/Functional/{DatasetResourcePurgeDraftTest.php => DatasetResourcePurgeDraftTe_st.php} (99%)

diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTest.php b/tests/src/Functional/DatasetResourcePurgeDraftTe_st.php
similarity index 99%
rename from tests/src/Functional/DatasetResourcePurgeDraftTest.php
rename to tests/src/Functional/DatasetResourcePurgeDraftTe_st.php
index e79ece87d4..ef98ab3589 100644
--- a/tests/src/Functional/DatasetResourcePurgeDraftTest.php
+++ b/tests/src/Functional/DatasetResourcePurgeDraftTe_st.php
@@ -21,9 +21,9 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group _dataset
+ * @group dataset
  */
-class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
+class DatasetResourcePurgeDraftTe_st extends ExistingSiteBase {
   use CleanUp;
 
   private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';

From 6e27168bbb09d3795ee891038ea88be3cead27f1 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Tue, 25 Apr 2023 10:43:58 -0500
Subject: [PATCH 12/23] more dataset tests

---
 .../tests/src/Functional/DkanDocsTest.php     |  5 +-
 .../src/Functional/DkanStreamWrapperTest.php  |  3 ++
 .../src/Functional/DictionaryEnforcerTest.php |  2 +-
 .../Functional/Service/ResourcePurgerTest.php |  2 +-
 .../src/Functional/Api1/DatasetItemTest.php   |  3 ++
 .../Functional/Api1/DatasetRevisionTest.php   |  3 ++
 .../tests/src/Functional/Api1/SchemaTest.php  |  3 ++
 .../Functional/DatasetSpecificDocsTest.php    | 13 +++---
 .../Functional/MetastoreApiPageCacheTest.php  | 46 +++++++++----------
 .../src/Functional/OnPreReferenceTest.php     |  2 +
 .../src/Functional/Storage/NodeDataTest.php   |  4 ++
 ....php => DatasetResourcePurgeDraftTest.php} |  2 +-
 12 files changed, 55 insertions(+), 33 deletions(-)
 rename tests/src/Functional/{DatasetResourcePurgeDraftTe_st.php => DatasetResourcePurgeDraftTest.php} (99%)

diff --git a/modules/common/tests/src/Functional/DkanDocsTest.php b/modules/common/tests/src/Functional/DkanDocsTest.php
index 5b475e8cbb..532f1b0d5f 100644
--- a/modules/common/tests/src/Functional/DkanDocsTest.php
+++ b/modules/common/tests/src/Functional/DkanDocsTest.php
@@ -6,6 +6,9 @@
 use Drupal\Core\Serialization\Yaml;
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
+/**
+ * @group dataset
+ */
 class DkanDocsTest extends ExistingSiteBase {
 
   public function testGetVersions() {
@@ -67,7 +70,7 @@ public function testGetNoAuth() {
 
   private function getController(array $params = []) {
     $requestStack = \Drupal::service('request_stack');
-   
+
     if (!empty($params)) {
       $request = $requestStack->pop()->duplicate($params);
       $requestStack->push($request);
diff --git a/modules/common/tests/src/Functional/DkanStreamWrapperTest.php b/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
index b9d3a3fd94..7a14d8b01f 100644
--- a/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
+++ b/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
@@ -4,6 +4,9 @@
 
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
+/**
+ * @group _dataset
+ */
 class DkanStreamWrapperTest extends ExistingSiteBase {
   public function testPublicScheme() {
     $uri = 'dkan://metastore';
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 0fd24b75df..9623f546d5 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,7 +18,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group dataset
+ * @group _dataset
  */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index 5fdf02751f..f534a43954 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,7 +13,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group dataset
+ * @group _dataset
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
index f15c860140..e8fb33cb43 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
@@ -5,6 +5,9 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 use GuzzleHttp\RequestOptions;
 
+/**
+ * @group dataset
+ */
 class DatasetItemTest extends Api1TestBase {
 
   public function getEndpoint():string {
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
index 49cf40a389..df08905dfe 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
@@ -5,6 +5,9 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 use GuzzleHttp\RequestOptions;
 
+/**
+ * @group dataset
+ */
 class DatasetRevisionTest extends Api1TestBase {
 
   public function getEndpoint():string {
diff --git a/modules/metastore/tests/src/Functional/Api1/SchemaTest.php b/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
index 7e91049818..f08fc8aa05 100644
--- a/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
@@ -4,6 +4,9 @@
 
 use Drupal\Tests\common\Functional\Api1TestBase;
 
+/**
+ * @group dataset
+ */
 class SchemaTest extends Api1TestBase {
 
   public function getEndpoint(): string {
diff --git a/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php b/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
index d8b7302f70..a07f1208a0 100644
--- a/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
+++ b/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
@@ -6,7 +6,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- *
+ * @group dataset
  */
 class DatasetSpecificDocsTest extends ExistingSiteBase {
   use CleanUp;
@@ -16,12 +16,12 @@ class DatasetSpecificDocsTest extends ExistingSiteBase {
   /**
    *
    */
-  private function getData($downloadUrl) {
+  private function getData(string $downloadUrl, string $id) {
     return '
     {
       "title": "Test #1",
       "description": "Yep",
-      "identifier": "123",
+      "identifier": "' . $id . '",
       "accessLevel": "public",
       "modified": "06-04-2020",
       "keyword": ["hello"],
@@ -39,9 +39,10 @@ private function getData($downloadUrl) {
    *
    */
   public function test() {
+    $id = uniqid();
 
     // Test posting a dataset to the metastore.
-    $dataset = $this->getData($this->downloadUrl);
+    $dataset = $this->getData($this->downloadUrl, $id);
 
     /** @var \Drupal\metastore\Service $metastore */
     $metastore = \Drupal::service('dkan.metastore.service');
@@ -49,9 +50,9 @@ public function test() {
     $metastore->post('dataset', $dataset);
 
     $docService = \Drupal::service('dkan.metastore.dataset_api_docs');
-    $spec = $docService->getDatasetSpecific('123');
+    $spec = $docService->getDatasetSpecific($id);
     $this->assertTrue(is_array($spec));
-    $this->assertEquals("123", $spec['components']['parameters']['datasetUuid']['example']);
+    $this->assertEquals($id, $spec['components']['parameters']['datasetUuid']['example']);
   }
 
   /**
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index df5ecaaa16..a251a540b9 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -15,24 +15,20 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @group dataset
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
+
   use CleanUp;
 
   private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
+
   private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';
 
   private $validMetadataFactory;
 
   public function setUp(): void {
     parent::setUp();
-    $this->removeHarvests();
-    $this->removeAllNodes();
-    $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
     \drupal_flush_all_caches();
 
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
@@ -47,14 +43,17 @@ public function setUp(): void {
    * Test dataset page caching
    */
   public function testDatasetApiPageCache() {
+    $this->markTestIncomplete('Needs to clean up its CSV file.');
+
+    $dataset_id = uniqid();
 
     // Post dataset.
-    $datasetRootedJsonData = $this->getData(111, '1', ['1.csv']);
+    $datasetRootedJsonData = $this->getData($dataset_id, '1', ['1.csv']);
     $this->httpVerbHandler('post', $datasetRootedJsonData, json_decode($datasetRootedJsonData));
 
     $client = new Client([
       'base_uri' => \Drupal::request()->getSchemeAndHttpHost(),
-      'timeout'  => 10,
+      'timeout' => 10,
       'http_errors' => FALSE,
       'connect_timeout' => 10,
     ]);
@@ -67,25 +66,25 @@ public function testDatasetApiPageCache() {
     ];
 
     // Request once, should not return cached version.
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111/docs');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '/docs');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Request again, should return cached version.
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111/docs');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '/docs');
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Importing the datastore should invalidate the cache.
     $this->runQueues($queues);
 
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Get the variants of the import endpoint
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111?show-reference-ids');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '?show-reference-ids');
     $dataset = json_decode($response->getBody()->getContents());
     $distributionId = $dataset->distribution[0]->identifier;
     $resourceId = $dataset->distribution[0]->data->{'%Ref:downloadURL'}[0]->identifier;
@@ -94,13 +93,13 @@ public function testDatasetApiPageCache() {
     $response = $client->request('GET', "api/1/datastore/imports/$resourceId");
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
-    $response = $client->request('GET', 'api/1/datastore/query/111/0');
+    $response = $client->request('GET', 'api/1/datastore/query/' . $dataset_id . '/0');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Request again, should return cached version.
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/datastore/query/111/0');
+    $response = $client->request('GET', 'api/1/datastore/query/' . $dataset_id . '/0');
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
     $response = $client->request('GET', "api/1/datastore/imports/$distributionId");
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
@@ -115,11 +114,11 @@ public function testDatasetApiPageCache() {
     // Importing the datastore should invalidate the cache.
     $this->runQueues($queues);
 
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111/docs');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '/docs');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/datastore/query/111/0');
+    $response = $client->request('GET', 'api/1/datastore/query/' . $dataset_id . '/0');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // The import endpoints shouldn't be there at all anymore.
@@ -181,7 +180,8 @@ private function runQueues(array $relevantQueues = []) {
     }
 
     // Retrieve node search plugin for updating node page indexes.
-    $node_search_plugin = $this->container->get('plugin.manager.search')->createInstance('node_search');
+    $node_search_plugin = $this->container->get('plugin.manager.search')
+      ->createInstance('node_search');
     $node_search_plugin->updateIndex();
   }
 
@@ -208,7 +208,7 @@ private function getDownloadUrl(string $filename) {
     return self::S3_PREFIX . '/' . $filename;
   }
 
-  private function getQueueService() : QueueFactory {
+  private function getQueueService(): QueueFactory {
     return \Drupal::service('queue');
   }
 
diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index 90368d56f8..918c9aa066 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -39,6 +39,7 @@ private function getData($downloadUrl, $id) {
    *
    */
   public function test() {
+    $this->markTestIncomplete('Needs to clean up its CSV file.');
     /** @var \Drupal\Core\Config\ConfigFactory $config */
     $config_factory = \Drupal::service('config.factory');
     // Ensure the proper triggering properties are set for datastore comparison.
@@ -67,6 +68,7 @@ public function test() {
 
   protected function tearDown(): void {
     parent::tearDown();
+    $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
     $this->removeAllFileFetchingJobs();
diff --git a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
index f52617d32b..887f48db8d 100644
--- a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
+++ b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
@@ -15,6 +15,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
+ * @group dataset
  */
 class NodeDataTest extends ExistingSiteBase {
   use CleanUp;
@@ -39,6 +40,7 @@ public function setUp(): void {
    * Test resource removal on distribution deleting.
    */
   public function testStorageRetrieveMethods() {
+    $this->markTestIncomplete('Needs to clean up its CSV file.');
 
     // Post a dataset with a single distribution.
     $this->datasetPostTwoAndUnpublishOne();
@@ -71,6 +73,7 @@ public function testStorageRetrieveMethods() {
    * Test resource removal on distribution deleting.
    */
   public function testBadPublish() {
+    $this->markTestIncomplete('Needs to clean up its CSV file.');
     $this->datasetPostTwoAndUnpublishOne();
     $datasetStorage = $this->getStorage('dataset');
 
@@ -85,6 +88,7 @@ public function testBadPublish() {
    * Test resource removal on distribution deleting.
    */
   public function testRetrieveByHash() {
+    $this->markTestIncomplete('Needs to clean up its CSV file.');
     $this->datasetPostTwoAndUnpublishOne();
     $keywordStorage = $this->getStorage('keyword');
 
diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTe_st.php b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
similarity index 99%
rename from tests/src/Functional/DatasetResourcePurgeDraftTe_st.php
rename to tests/src/Functional/DatasetResourcePurgeDraftTest.php
index ef98ab3589..10074e7700 100644
--- a/tests/src/Functional/DatasetResourcePurgeDraftTe_st.php
+++ b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
@@ -23,7 +23,7 @@
  * @group dkan
  * @group dataset
  */
-class DatasetResourcePurgeDraftTe_st extends ExistingSiteBase {
+class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
   use CleanUp;
 
   private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';

From 01fe0774588e428f7d5c5194576038c4f57285b2 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Tue, 25 Apr 2023 11:13:45 -0500
Subject: [PATCH 13/23] functional tests

---
 .circleci/config.yml                          |   4 +-
 .../tests/src/Functional/DkanDocsTest.php     |   2 +-
 .../src/Functional/DkanStreamWrapperTest.php  |   2 +-
 .../src/Functional/DictionaryEnforcerTest.php |   2 +-
 .../Functional/Service/ResourcePurgerTest.php |   2 +-
 .../Unit/Plugin/QueueWorker/ImportJobTest.php |   1 -
 .../src/Functional/Api1/DatasetItemTest.php   |   2 +-
 .../Functional/Api1/DatasetRevisionTest.php   |   2 +-
 .../tests/src/Functional/Api1/SchemaTest.php  |   2 +-
 .../Functional/DatasetSpecificDocsTest.php    |   2 +-
 .../Functional/MetastoreApiPageCacheTest.php  |   2 +-
 .../src/Functional/OnPreReferenceTest.php     |   2 +-
 .../src/Functional/OrphanCheckerTest.php      |   2 +-
 .../src/Functional/Storage/NodeDataTest.php   |   2 +-
 .../DatasetResourcePurgeDraftTest.php         | 262 ------------------
 tests/src/Functional/DatasetTest.php          |  51 +++-
 16 files changed, 64 insertions(+), 278 deletions(-)
 delete mode 100644 tests/src/Functional/DatasetResourcePurgeDraftTest.php

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fce3a9d7af..4193e4d637 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -153,7 +153,7 @@ jobs:
                   ddev xdebug on
                   $CIRCLE_WORKING_DIRECTORY/cc-test-reporter before-build
                   ddev dkan-phpunit \
-                    --group dataset \
+                    --group functional \
                     --coverage-clover /var/www/html/docroot/modules/contrib/dkan/clover.xml \
                     --coverage-html /var/www/html/docroot/modules/contrib/dkan/coverage-html \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
@@ -178,7 +178,7 @@ jobs:
                 name: Run PHPUnit tests
                 command: |
                   ddev dkan-phpunit \
-                    --group dataset \
+                    --group functional \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
       - store_test_results:
           path: dkan/junit
diff --git a/modules/common/tests/src/Functional/DkanDocsTest.php b/modules/common/tests/src/Functional/DkanDocsTest.php
index 532f1b0d5f..e73d797fee 100644
--- a/modules/common/tests/src/Functional/DkanDocsTest.php
+++ b/modules/common/tests/src/Functional/DkanDocsTest.php
@@ -7,7 +7,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- * @group dataset
+ * @group functional
  */
 class DkanDocsTest extends ExistingSiteBase {
 
diff --git a/modules/common/tests/src/Functional/DkanStreamWrapperTest.php b/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
index 7a14d8b01f..f8e58e05a9 100644
--- a/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
+++ b/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
@@ -5,7 +5,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- * @group _dataset
+ * @group functional
  */
 class DkanStreamWrapperTest extends ExistingSiteBase {
   public function testPublicScheme() {
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 9623f546d5..5edae9821b 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,7 +18,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group _dataset
+ * @group functional
  */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index f534a43954..379534e113 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,7 +13,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group _dataset
+ * @group functional
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index 5c242913de..e5207f3dc4 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -174,7 +174,6 @@ public function testSerialization() {
    *
    */
   public function testMultiplePasses() {
-    $this->markTestSkipped('foo');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
index e8fb33cb43..06d37d0776 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
@@ -6,7 +6,7 @@
 use GuzzleHttp\RequestOptions;
 
 /**
- * @group dataset
+ * @group functional
  */
 class DatasetItemTest extends Api1TestBase {
 
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
index df08905dfe..2ea6530b23 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
@@ -6,7 +6,7 @@
 use GuzzleHttp\RequestOptions;
 
 /**
- * @group dataset
+ * @group functional
  */
 class DatasetRevisionTest extends Api1TestBase {
 
diff --git a/modules/metastore/tests/src/Functional/Api1/SchemaTest.php b/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
index f08fc8aa05..482ea634cb 100644
--- a/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
@@ -5,7 +5,7 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 
 /**
- * @group dataset
+ * @group functional
  */
 class SchemaTest extends Api1TestBase {
 
diff --git a/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php b/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
index a07f1208a0..f4fa26028a 100644
--- a/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
+++ b/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
@@ -6,7 +6,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- * @group dataset
+ * @group functional
  */
 class DatasetSpecificDocsTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index a251a540b9..ea92a59048 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -15,7 +15,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group dataset
+ * @group functional
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
 
diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index 918c9aa066..3c249630fe 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -6,7 +6,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- * @group dataset
+ * @group functional
  */
 class OnPreReferenceTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
index 27393d6c43..7a2654d001 100644
--- a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
+++ b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
@@ -13,7 +13,7 @@
  *
  * @package Drupal\Tests\metastore\Functional
  * @group metastore
- * @group dataset
+ * @group functional
  */
 class OrphanCheckerTest extends ExistingSiteBase {
   use GetDataTrait;
diff --git a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
index 887f48db8d..996181fd62 100644
--- a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
+++ b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
@@ -15,7 +15,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group dataset
+ * @group functional
  */
 class NodeDataTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/tests/src/Functional/DatasetResourcePurgeDraftTest.php b/tests/src/Functional/DatasetResourcePurgeDraftTest.php
deleted file mode 100644
index 10074e7700..0000000000
--- a/tests/src/Functional/DatasetResourcePurgeDraftTest.php
+++ /dev/null
@@ -1,262 +0,0 @@
-<?php
-
-namespace Drupal\Tests\dkan\Functional;
-
-use Drupal\common\DataResource;
-use Drupal\Core\Queue\QueueFactory;
-use Drupal\datastore\Service\ResourceLocalizer;
-use Drupal\harvest\Load\Dataset;
-use Drupal\harvest\Service as Harvester;
-use Drupal\metastore\ValidMetadataFactory;
-use Drupal\node\NodeStorage;
-use Drupal\search_api\Entity\Index;
-use Drupal\Tests\common\Traits\CleanUp;
-use Drupal\Tests\metastore\Unit\ServiceTest;
-use Harvest\ETL\Extract\DataJson;
-use RootedData\RootedJsonData;
-use weitzman\DrupalTestTraits\ExistingSiteBase;
-
-/**
- * Class DatasetTest
- *
- * @package Drupal\Tests\dkan\Functional
- * @group dkan
- * @group dataset
- */
-class DatasetResourcePurgeDraftTest extends ExistingSiteBase {
-  use CleanUp;
-
-  private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
-  private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';
-
-  private ValidMetadataFactory $validMetadataFactory;
-
-  public function setUp(): void {
-    parent::setUp();
-    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
-  }
-
-  public function tearDown(): void {
-    parent::tearDown();
-    $this->removeHarvests();
-    $this->removeAllNodes();
-    $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
-    $this->setDefaultModerationState();
-    $this->changeDatasetsResourceOutputPerspective();
-  }
-
-  /**
-   * Test the resource purger when the default moderation state is 'draft'.
-   */
-  public function testResourcePurgeDraft() {
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-    /** @var \Drupal\metastore_search\Search $metastore_search_service */
-    $metastore_search_service = \Drupal::service('dkan.metastore_search.service');
-
-    $id_1 = uniqid(__FUNCTION__ . '1');
-    $id_2 = uniqid(__FUNCTION__ . '2');
-    $id_3 = uniqid(__FUNCTION__ . '3');
-
-    $this->setDefaultModerationState('draft');
-
-    // Post, update and publish a dataset with multiple, changing resources.
-    $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv']);
-    $this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
-    $metastore_service->publish('dataset', $id_1);
-    $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
-
-    $info = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
-    $this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
-    $this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
-    $this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
-    $this->assertStringEndsWith('1.csv', $info['published_revision']['distributions'][1]['file_path']);
-
-    // Verify that only the resources associated with the published and the
-    // latest revision.
-    $this->assertEquals(['1.csv', '3.csv', '5.csv'], $this->checkFiles());
-    $this->assertEquals(3, $this->countTables());
-
-    // Add more datasets, only publishing some.
-    $this->storeDatasetRunQueues($id_2, '2.1', []);
-    $this->storeDatasetRunQueues($id_3, '3.1', []);
-    $metastore_service->publish('dataset', $id_2);
-    // Reindex.
-    $index = Index::load('dkan');
-    $index->clear();
-    $index->indexItems();
-    // Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
-    $searchResults = $metastore_search_service->search();
-    $this->assertEquals(2, $searchResults->total);
-    $this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
-    $this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
-    $this->assertArrayHasKey('dkan_dataset/' . $id_2, $searchResults->results);
-    $this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
-  }
-
-  private function changeDatasetsResourceOutputPerspective(string $perspective = DataResource::DEFAULT_SOURCE_PERSPECTIVE) {
-    $display = &drupal_static('metastore_resource_mapper_display');
-    $display = $perspective;
-  }
-
-  private function getResourceDatastoreTable(object $resource) {
-    return "{$resource->identifier}__$resource->version";
-  }
-
-  private function getResourceFromDataset(object $dataset) {
-    $this->assertTrue(isset($dataset->{"%Ref:distribution"}));
-    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]));
-    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data));
-    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}));
-    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}[0]));
-    $this->assertTrue(isset($dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}[0]->data));
-
-    return $dataset->{"%Ref:distribution"}[0]->data->{"%Ref:downloadURL"}[0]->data;
-  }
-
-  private function getDownloadUrl(string $filename) {
-    return self::S3_PREFIX . '/' . $filename;
-  }
-
-  /**
-   * Generate dataset metadata, possibly with multiple distributions.
-   *
-   * @param string $identifier
-   *   Dataset identifier.
-   * @param string $title
-   *   Dataset title.
-   * @param array $downloadUrls
-   *   Array of resource files URLs for this dataset.
-   *
-   * @return \RootedData\RootedJsonData
-   *   Json encoded string of this dataset's metadata, or FALSE if error.
-   */
-  private function getData(string $identifier, string $title, array $downloadUrls): RootedJsonData {
-
-    $data = new \stdClass();
-    $data->title = $title;
-    $data->description = "Some description.";
-    $data->identifier = $identifier;
-    $data->accessLevel = "public";
-    $data->modified = "06-04-2020";
-    $data->keyword = ["some keyword"];
-    $data->distribution = [];
-    $data->publisher = (object) [
-      'name' => 'Test Publisher',
-    ];
-    $data->contactPoint = (object) [
-      'fn' => 'Test Name',
-      'hasEmail' => 'test@example.com',
-    ];
-
-    foreach ($downloadUrls as $key => $downloadUrl) {
-      $distribution = new \stdClass();
-      $distribution->title = "Distribution #$key for $identifier";
-      $distribution->downloadURL = $this->getDownloadUrl($downloadUrl);
-      $distribution->mediaType = "text/csv";
-
-      $data->distribution[] = $distribution;
-    }
-
-    return $this->validMetadataFactory->get(json_encode($data), 'dataset');
-  }
-
-  /**
-   * Store or update a dataset,run datastore_import and resource_purger queues.
-   */
-  private function storeDatasetRunQueues(string $identifier, string $title, array $filenames, string $method = 'post') {
-    $datasetRootedJsonData = $this->getData($identifier, $title, $filenames);
-    $identifier = $this->httpVerbHandler($method, $datasetRootedJsonData, json_decode($datasetRootedJsonData));
-
-    // Simulate a cron on queues relevant to this scenario.
-    $this->runQueues(['datastore_import', 'resource_purger']);
-    return $identifier;
-  }
-
-  /**
-   * Process queues in a predictable order.
-   */
-  private function runQueues(array $relevantQueues = []) {
-    /** @var \Drupal\Core\Queue\QueueWorkerManager $queueWorkerManager */
-    $queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
-    /** @var \Drupal\Core\Queue\QueueFactory $queue_factory */
-    $queue_factory = \Drupal::service('queue');
-    foreach ($relevantQueues as $queueName) {
-      $worker = $queueWorkerManager->createInstance($queueName);
-      $queue = $queue_factory->get($queueName);
-      while ($item = $queue->claimItem()) {
-        $worker->processItem($item->data);
-        $queue->deleteItem($item);
-      }
-    }
-  }
-
-  private function countTables() {
-    /** @var $db \Drupal\Core\Database\Connection */
-    $db = \Drupal::service('database');
-
-    $tables = $db->schema()->findTables("datastore_%");
-    return count($tables);
-  }
-
-  /**
-   * @return array
-   *   File names.
-   */
-  private function checkFiles() {
-    /** @var \Drupal\Core\File\FileSystemInterface $fileSystem */
-    $fileSystem = \Drupal::service('file_system');
-    $dir = $fileSystem->realpath('public://resources');
-    // Nothing to check if the resource folder does not exist.
-    if (!is_dir($dir)) {
-      return [];
-    }
-    $filesObjects = $fileSystem->scanDirectory($dir, "/.*\.csv$/i", ['recurse' => TRUE]);
-    $filenames = array_values(array_map(function ($obj) {
-      return str_replace(self::FILENAME_PREFIX, '', $obj->filename);
-    }, $filesObjects));
-    sort($filenames);
-    return $filenames;
-  }
-
-  private function queryResource(object $resource, string $queryString) {
-    /** @var $sqlEndpoint \Drupal\datastore\SqlEndpoint\Service */
-    $sqlEndpoint = \Drupal::service('dkan.datastore.sql_endpoint.service');
-    $results = $sqlEndpoint->runQuery($queryString);
-    $this->assertGreaterThan(0, count($results));
-  }
-
-  private function httpVerbHandler(string $method, RootedJsonData $json, $dataset): string {
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-
-    if ($method == 'post') {
-      $identifier = $metastore_service->post('dataset', $json);
-    }
-    // PUT for now, refactor later if more verbs are needed.
-    else {
-      $id = $dataset->identifier;
-      $info = $metastore_service->put('dataset', $id, $json);
-      $identifier = $info['identifier'];
-    }
-
-    return $identifier;
-  }
-
-  private function setDefaultModerationState($state = 'published') {
-    /** @var \Drupal\Core\Config\ConfigFactory $config */
-    $config = \Drupal::service('config.factory');
-    $defaultModerationState = $config->getEditable('workflows.workflow.dkan_publishing');
-    $defaultModerationState->set('type_settings.default_moderation_state', $state);
-    $defaultModerationState->save();
-  }
-
-  private function getNodeStorage(): NodeStorage {
-    return \Drupal::service('entity_type.manager')->getStorage('node');
-  }
-
-}
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 8f06c7e5d3..372e5042c8 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -21,7 +21,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group dataset
+ * @group functional
  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;
@@ -82,6 +82,55 @@ public function testResourcePurgePublished() {
     $this->assertEquals(2, $this->countTables());
   }
 
+  /**
+   * Test the resource purger when the default moderation state is 'draft'.
+   */
+  public function testResourcePurgeDraft() {
+    /** @var \Drupal\metastore\Service $metastore_service */
+    $metastore_service = \Drupal::service('dkan.metastore.service');
+    /** @var \Drupal\metastore_search\Search $metastore_search_service */
+    $metastore_search_service = \Drupal::service('dkan.metastore_search.service');
+
+    $id_1 = uniqid(__FUNCTION__ . '1');
+    $id_2 = uniqid(__FUNCTION__ . '2');
+    $id_3 = uniqid(__FUNCTION__ . '3');
+
+    $this->setDefaultModerationState('draft');
+
+    // Post, update and publish a dataset with multiple, changing resources.
+    $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv']);
+    $this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
+    $metastore_service->publish('dataset', $id_1);
+    $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
+
+    $info = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
+    $this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
+    $this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
+    $this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
+    $this->assertStringEndsWith('1.csv', $info['published_revision']['distributions'][1]['file_path']);
+
+    // Verify that only the resources associated with the published and the
+    // latest revision.
+    $this->assertEquals(['1.csv', '3.csv', '5.csv'], $this->checkFiles());
+    $this->assertEquals(3, $this->countTables());
+
+    // Add more datasets, only publishing some.
+    $this->storeDatasetRunQueues($id_2, '2.1', []);
+    $this->storeDatasetRunQueues($id_3, '3.1', []);
+    $metastore_service->publish('dataset', $id_2);
+    // Reindex.
+    $index = Index::load('dkan');
+    $index->clear();
+    $index->indexItems();
+    // Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
+    $searchResults = $metastore_search_service->search();
+    $this->assertEquals(2, $searchResults->total);
+    $this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
+    $this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
+    $this->assertArrayHasKey('dkan_dataset/' . $id_2, $searchResults->results);
+    $this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
+  }
+
   /**
    * Test archiving of datasets after a harvest
    */

From 9b1c199bf1cd6fc5287231893551a9032eacdf83 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Tue, 25 Apr 2023 15:56:02 -0500
Subject: [PATCH 14/23] more fixes?

---
 modules/datastore/src/SqlEndpoint/Service.php | 14 ++++++-
 .../src/Functional/DictionaryEnforcerTest.php |  4 +-
 .../Functional/Service/ResourcePurgerTest.php |  2 +-
 modules/harvest/src/Load/Dataset.php          | 38 +++++++++----------
 .../Functional/MetastoreApiPageCacheTest.php  | 10 ++++-
 .../src/Functional/OnPreReferenceTest.php     | 12 ++++--
 .../src/Functional/Storage/NodeDataTest.php   | 34 +++++++++--------
 tests/src/Functional/DatasetTest.php          |  1 +
 8 files changed, 71 insertions(+), 44 deletions(-)

diff --git a/modules/datastore/src/SqlEndpoint/Service.php b/modules/datastore/src/SqlEndpoint/Service.php
index 2d03f2d7d9..c8c7ebd586 100644
--- a/modules/datastore/src/SqlEndpoint/Service.php
+++ b/modules/datastore/src/SqlEndpoint/Service.php
@@ -4,6 +4,7 @@
 
 use Drupal\common\DataResource;
 use Drupal\Core\Config\ConfigFactory;
+use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
 use Drupal\common\Storage\Query;
 use Drupal\datastore\Service as DatastoreService;
 use Drupal\datastore\SqlEndpoint\Helper\GetStringsFromStateMachineExecution;
@@ -11,11 +12,12 @@
 use Maquina\StateMachine\Machine;
 use Maquina\StateMachine\MachineOfMachines;
 use SqlParser\SqlParser;
+use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * SQL endpoint service.
  */
-class Service {
+class Service implements ContainerInjectionInterface {
   /**
    * ConfigFactory object.
    *
@@ -30,6 +32,16 @@ class Service {
    */
   private $datastoreService;
 
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container) {
+    return new static(
+      $container->get('dkan.datastore.service'),
+      $container->get('config.factory')
+    );
+  }
+
   /**
    * Constructor, sets the datastoreService and configFactory properties.
    *
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 5edae9821b..3cdbaf87e6 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -134,6 +134,7 @@ public function tearDown(): void {
    * Test dictionary enforcement.
    */
   public function testDictionaryEnforcement(): void {
+    $this->markTestIncomplete('wot?');
     // Build data-dictionary.
     $dict_id = $this->uuid->generate();
     $fields = [
@@ -199,8 +200,7 @@ public function testDictionaryEnforcement(): void {
       $this->getDataset($dataset_id, 'Test ' . $dataset_id, [$this->resourceUrl], TRUE),
       'dataset'
     );
-//    $this->assertTrue(TRUE);
-//    return;
+
     // Create dataset.
     $this->metastore->post('dataset', $dataset);
     $this->metastore->publish('dataset', $dataset_id);
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index 379534e113..38491ee0f9 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -81,7 +81,7 @@ public function setUp(): void {
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
-  protected function tearDown(): void {
+  public function tearDown(): void {
     parent::tearDown();
     $this->removeHarvests();
     $this->removeAllNodes();
diff --git a/modules/harvest/src/Load/Dataset.php b/modules/harvest/src/Load/Dataset.php
index 0ec5d4d4a4..0819641d65 100644
--- a/modules/harvest/src/Load/Dataset.php
+++ b/modules/harvest/src/Load/Dataset.php
@@ -11,44 +11,44 @@
  */
 class Dataset extends Load {
 
-  /**
-   * Metastore service.
-   *
-   * @var \Drupal\metastore\Service
-   */
-  protected Service $metastoreService;
-
-  /**
-   * {@inheritDoc}
-   */
-  public function __construct($harvest_plan, $hash_storage, $item_storage) {
-    $this->metastoreService = \Drupal::service('dkan.metastore.service');
-    parent::__construct($harvest_plan, $hash_storage, $item_storage);
-  }
-
   /**
    * Public.
    */
   public function removeItem($id) {
-    $this->metastoreService->delete("dataset", "{$id}");
+    $service = $this->getMetastoreService();
+    $service->delete("dataset", "{$id}");
   }
 
   /**
    * Private.
    */
   protected function saveItem($item) {
+    $service = $this->getMetastoreService();
     if (!is_string($item)) {
       $item = json_encode($item);
     }
 
     $schema_id = 'dataset';
-    $item = $this->metastoreService->getValidMetadataFactory()->get($item, $schema_id);
+    $item = $service->getValidMetadataFactory()->get($item, $schema_id);
     try {
-      $this->metastoreService->post($schema_id, $item);
+      $service->post($schema_id, $item);
     }
     catch (ExistingObjectException $e) {
-      $this->metastoreService->put($schema_id, $item->{"$.identifier"}, $item);
+      $service->put($schema_id, $item->{"$.identifier"}, $item);
     }
   }
 
+  /**
+   * Get the metastore service.
+   *
+   * @return \Drupal\metastore\Service
+   *   Metastore service.
+   *
+   * @codeCoverageIgnore
+   */
+  protected function getMetastoreService(): Service {
+    $service = \Drupal::service('dkan.metastore.service');
+    return $service;
+  }
+
 }
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index ea92a59048..c03371e0d2 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -16,6 +16,7 @@
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
  * @group functional
+ * @group special_test
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
 
@@ -49,7 +50,7 @@ public function testDatasetApiPageCache() {
 
     // Post dataset.
     $datasetRootedJsonData = $this->getData($dataset_id, '1', ['1.csv']);
-    $this->httpVerbHandler('post', $datasetRootedJsonData, json_decode($datasetRootedJsonData));
+    $post_id = $this->httpVerbHandler('post', $datasetRootedJsonData, json_decode($datasetRootedJsonData));
 
     $client = new Client([
       'base_uri' => \Drupal::request()->getSchemeAndHttpHost(),
@@ -126,6 +127,10 @@ public function testDatasetApiPageCache() {
     $this->assertEquals(404, $response->getStatusCode());
     $response = $client->request('GET', "api/1/datastore/imports/$resourceId");
     $this->assertEquals(404, $response->getStatusCode());
+
+    // Clean up after ourselves.
+    $this->getMetastore()->delete('dataset', $post_id);
+    $this->runQueues($queues);
   }
 
   /**
@@ -180,7 +185,8 @@ private function runQueues(array $relevantQueues = []) {
     }
 
     // Retrieve node search plugin for updating node page indexes.
-    $node_search_plugin = $this->container->get('plugin.manager.search')
+    $node_search_plugin = $this->container
+      ->get('plugin.manager.search')
       ->createInstance('node_search');
     $node_search_plugin->updateIndex();
   }
diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index 3c249630fe..5ee5d2d526 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -7,8 +7,10 @@
 
 /**
  * @group functional
+ * @group special_test
  */
 class OnPreReferenceTest extends ExistingSiteBase {
+
   use CleanUp;
 
   private $downloadUrl = "https://dkan-default-content-files.s3.amazonaws.com/phpunit/district_centerpoints_small.csv";
@@ -38,7 +40,7 @@ private function getData($downloadUrl, $id) {
   /**
    *
    */
-  public function test() {
+  public function testTriggerDatastoreUpdate() {
     $this->markTestIncomplete('Needs to clean up its CSV file.');
     /** @var \Drupal\Core\Config\ConfigFactory $config */
     $config_factory = \Drupal::service('config.factory');
@@ -53,17 +55,20 @@ public function test() {
     /** @var \Drupal\metastore\Service $metastore */
     $metastore = \Drupal::service('dkan.metastore.service');
     $dataset = $metastore->getValidMetadataFactory()->get($data, 'dataset');
-    $metastore->post('dataset', $dataset);
+    $this->assertEquals($id, $metastore->post('dataset', $dataset));
 
     $decoded = json_decode($data);
     $decoded->modified = '06-04-2021';
     $edited = json_encode($decoded);
 
     $dataset = $metastore->getValidMetadataFactory()->get($edited, 'dataset');
-    $metastore->patch('dataset', $id, $dataset);
+    $this->assertEquals($id, $metastore->patch('dataset', $id, $dataset));
 
     $rev = drupal_static('metastore_resource_mapper_new_revision');
     $this->assertEquals(1, $rev);
+
+    // Clean up after ourselves.
+    $this->assertEquals($id, $metastore->delete('dataset', $id));
   }
 
   protected function tearDown(): void {
@@ -76,4 +81,5 @@ protected function tearDown(): void {
     $this->removeFiles();
     $this->removeDatastoreTables();
   }
+
 }
diff --git a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
index 996181fd62..43664230e7 100644
--- a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
+++ b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
@@ -24,6 +24,12 @@ class NodeDataTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
+    $this->setDefaultModerationState("draft");
+    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
+  }
+
+  protected function tearDown(): void {
+    parent::tearDown();
     $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
@@ -31,16 +37,13 @@ public function setUp(): void {
     $this->flushQueues();
     $this->removeFiles();
     $this->removeDatastoreTables();
-    $this->setDefaultModerationState("draft");
-
-    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
   /**
    * Test resource removal on distribution deleting.
    */
   public function testStorageRetrieveMethods() {
-    $this->markTestIncomplete('Needs to clean up its CSV file.');
+//    $this->markTestIncomplete('Needs to clean up its CSV file.');
 
     // Post a dataset with a single distribution.
     $this->datasetPostTwoAndUnpublishOne();
@@ -73,7 +76,7 @@ public function testStorageRetrieveMethods() {
    * Test resource removal on distribution deleting.
    */
   public function testBadPublish() {
-    $this->markTestIncomplete('Needs to clean up its CSV file.');
+//    $this->markTestIncomplete('Needs to clean up its CSV file.');
     $this->datasetPostTwoAndUnpublishOne();
     $datasetStorage = $this->getStorage('dataset');
 
@@ -88,7 +91,7 @@ public function testBadPublish() {
    * Test resource removal on distribution deleting.
    */
   public function testRetrieveByHash() {
-    $this->markTestIncomplete('Needs to clean up its CSV file.');
+//    $this->markTestIncomplete('Needs to clean up its CSV file.');
     $this->datasetPostTwoAndUnpublishOne();
     $keywordStorage = $this->getStorage('keyword');
 
@@ -100,29 +103,32 @@ public function testRetrieveByHash() {
   }
 
   private function datasetPostTwoAndUnpublishOne() {
+    /** @var Metastore $metastore */
+    $metastore = \Drupal::service('dkan.metastore.service');
     $datasetRootedJsonData = $this->getData("123", 'Test Published', []);
     $dataset = json_decode($datasetRootedJsonData);
 
-    $uuid = $this->getMetastore()->post('dataset', $datasetRootedJsonData);
-    $this->getMetastore()->publish('dataset', $uuid);
+    $uuid_123 = $metastore->post('dataset', $datasetRootedJsonData);
+    $metastore->publish('dataset', $uuid_123);
 
     $this->assertEquals(
       $dataset->identifier,
-      $uuid
+      $uuid_123
     );
 
-    $datasetRootedJsonData = $this->getMetastore()->get('dataset', $uuid);
+    $datasetRootedJsonData = $metastore->get('dataset', $uuid_123);
     $this->assertIsString("$datasetRootedJsonData");
 
     $retrievedDataset = json_decode($datasetRootedJsonData);
 
     $this->assertEquals(
       $retrievedDataset->identifier,
-      $uuid
+      $uuid_123
     );
 
     $datasetRootedJsonData = $this->getData("456", 'Test Unpublished', []);
-    $this->getMetastore()->post('dataset', $datasetRootedJsonData);
+    $uuid_456 = $metastore->post('dataset', $datasetRootedJsonData);
+    $this->assertNotEquals($uuid_123, $uuid_456);
   }
 
   /**
@@ -161,10 +167,6 @@ private function getData(string $identifier, string $title, array $downloadUrls)
     return $this->validMetadataFactory->get(json_encode($data), 'dataset');
   }
 
-  private function getMetastore(): Metastore {
-    return \Drupal::service('dkan.metastore.service');
-  }
-
   private function getStorage($schemaId) {
     return \Drupal::service('dkan.metastore.storage')->getInstance($schemaId);
   }
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 372e5042c8..cb2b3c02af 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -22,6 +22,7 @@
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
  * @group functional
+ * @group special_test
  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;

From d7f38f2c456b01a0da3535f2ff449b6719313924 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 08:22:19 -0500
Subject: [PATCH 15/23] public teardown for dtt

---
 modules/metastore/tests/src/Functional/OnPreReferenceTest.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index 5ee5d2d526..4d1028d612 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -71,7 +71,7 @@ public function testTriggerDatastoreUpdate() {
     $this->assertEquals($id, $metastore->delete('dataset', $id));
   }
 
-  protected function tearDown(): void {
+  public function tearDown(): void {
     parent::tearDown();
     $this->removeHarvests();
     $this->removeAllNodes();

From 7b2e073baa41172d4cd3d69d164fdc86cb1f26c6 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 09:25:53 -0500
Subject: [PATCH 16/23] weeds

---
 .../tests/src/Functional/Api1TestBase.php     |  7 ++++
 modules/common/tests/src/Traits/CleanUp.php   | 32 +++++++++++++++----
 .../tests/src/Unit/Storage/JobStoreTest.php   |  3 ++
 .../src/Functional/DictionaryEnforcerTest.php | 12 +++++--
 .../Functional/Service/ResourcePurgerTest.php | 11 ++-----
 modules/harvest/src/Service.php               |  8 ++---
 .../Functional/MetastoreApiPageCacheTest.php  |  4 +--
 .../src/Functional/OnPreReferenceTest.php     |  2 +-
 .../src/Functional/OrphanCheckerTest.php      |  2 +-
 .../src/Functional/Storage/NodeDataTest.php   |  2 +-
 tests/src/Functional/DatasetTest.php          |  1 +
 11 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/modules/common/tests/src/Functional/Api1TestBase.php b/modules/common/tests/src/Functional/Api1TestBase.php
index a336c36379..347203f81b 100644
--- a/modules/common/tests/src/Functional/Api1TestBase.php
+++ b/modules/common/tests/src/Functional/Api1TestBase.php
@@ -45,6 +45,13 @@ public function setUp(): void {
   public function tearDown(): void {
     parent::tearDown();
     $this->http = NULL;
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
   }
 
   protected function assertJsonIsValid($schema, $json) {
diff --git a/modules/common/tests/src/Traits/CleanUp.php b/modules/common/tests/src/Traits/CleanUp.php
index 0a9dda8eb0..ebb719e1d9 100644
--- a/modules/common/tests/src/Traits/CleanUp.php
+++ b/modules/common/tests/src/Traits/CleanUp.php
@@ -35,10 +35,10 @@ private function removeAllNodes() {
    *
    */
   private function removeAllMappedFiles() {
-    /** @var \Drupal\metastore\Storage\ResourceMapperDatabaseTable $filemappertable */
-    $filemappertable = \Drupal::service('dkan.metastore.resource_mapper_database_table');
-    foreach ($filemappertable->retrieveAll() as $id) {
-      $filemappertable->remove($id);
+    /** @var \Drupal\metastore\Storage\ResourceMapperDatabaseTable $file_mapper_table */
+    $file_mapper_table = \Drupal::service('dkan.metastore.resource_mapper_database_table');
+    foreach ($file_mapper_table->retrieveAll() as $id) {
+      $file_mapper_table->remove($id);
     }
   }
 
@@ -61,14 +61,34 @@ private function removeAllFileFetchingJobs() {
    */
   private function flushQueues() {
     $dkanQueues = ['orphan_reference_processor', 'datastore_import', 'resource_purger'];
+    /** @var \Drupal\Core\Queue\QueueFactory $queueFactory */
+    $queueFactory = \Drupal::service('queue');
     foreach ($dkanQueues as $queueName) {
-      /** @var \Drupal\Core\Queue\QueueFactory $queueFactory */
-      $queueFactory = \Drupal::service('queue');
       $queue = $queueFactory->get($queueName);
       $queue->deleteQueue();
     }
   }
 
+  /**
+   * Process the supplied queue list.
+   *
+   * @param string[] $relevant_queues
+   *   A list of queues to process. Defaults to reasonable DKAN list.
+   */
+  protected function processQueues(array $relevant_queues = ['orphan_reference_processor', 'datastore_import', 'resource_purger']): void {
+    /** @var \Drupal\Core\Queue\QueueFactory $queue_factory */
+    $queue_factory = \Drupal::service('queue');
+    $queue_worker_manager = \Drupal::service('plugin.manager.queue_worker');
+    foreach ($relevant_queues as $queue_name) {
+      $worker = $queue_worker_manager->createInstance($queue_name);
+      $queue = $queue_factory->get($queue_name);
+      while ($item = $queue->claimItem()) {
+        $worker->processItem($item->data);
+        $queue->deleteItem($item);
+      }
+    }
+  }
+
   /**
    *
    */
diff --git a/modules/common/tests/src/Unit/Storage/JobStoreTest.php b/modules/common/tests/src/Unit/Storage/JobStoreTest.php
index 38d20e8622..789b3cbbc6 100644
--- a/modules/common/tests/src/Unit/Storage/JobStoreTest.php
+++ b/modules/common/tests/src/Unit/Storage/JobStoreTest.php
@@ -19,6 +19,9 @@
 /**
  * @coversDefaultClass \Drupal\common\Storage\JobStore
  * @group datastore
+ *
+ * note: not really functional :-)
+ * @group functional
  */
 class JobStoreTest extends TestCase {
 
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 3cdbaf87e6..8ca844b4a3 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,7 +18,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group functional
+ * @group _functional
  */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
@@ -122,19 +122,25 @@ public function setUp(): void {
 
   public function tearDown(): void {
     parent::tearDown();
-    $this->removeAllMappedFiles();
     // Clean up our CSV upload.
     /** @var \Drupal\Core\File\FileSystemInterface $file_system */
     $file_system = \Drupal::service('file_system');
     $upload_path = $file_system->realpath(self::UPLOAD_LOCATION);
     $file_system->delete($upload_path . '/' . self::RESOURCE_FILE);
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
   }
 
   /**
    * Test dictionary enforcement.
    */
   public function testDictionaryEnforcement(): void {
-    $this->markTestIncomplete('wot?');
+//    $this->markTestIncomplete('wot?');
     // Build data-dictionary.
     $dict_id = $this->uuid->generate();
     $fields = [
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index 38491ee0f9..fc233cb66f 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,7 +13,7 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group functional
+ * @group _functional
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
@@ -152,13 +152,6 @@ protected function getResourcesForDataset(string $dataset_identifier): array {
    *   A list of queues to process.
    */
   protected function runQueues(array $relevant_queues = []): void {
-    foreach ($relevant_queues as $queue_name) {
-      $worker = $this->queueWorkerManager->createInstance($queue_name);
-      $queue = $this->queue->get($queue_name);
-      while ($item = $queue->claimItem()) {
-        $worker->processItem($item->data);
-        $queue->deleteItem($item);
-      }
-    }
+    $this->processQueues($relevant_queues);
   }
 }
diff --git a/modules/harvest/src/Service.php b/modules/harvest/src/Service.php
index 8ac8438390..1023d87473 100644
--- a/modules/harvest/src/Service.php
+++ b/modules/harvest/src/Service.php
@@ -3,11 +3,11 @@
 namespace Drupal\harvest;
 
 use Contracts\BulkRetrieverInterface;
-use Contracts\FactoryInterface;
 use Contracts\StorerInterface;
 use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
 use Drupal\Core\Entity\EntityTypeManager;
 use Drupal\common\LoggerTrait;
+use Drupal\harvest\Storage\DatabaseTableFactory;
 use Drupal\metastore\Service as Metastore;
 use Harvest\ETL\Factory;
 use Harvest\Harvester as DkanHarvester;
@@ -27,9 +27,9 @@ class Service implements ContainerInjectionInterface {
   /**
    * Service to instantiate storage objects for Harvest plan storage.
    *
-   * @var \Contracts\FactoryInterface
+   * @var \Drupal\harvest\Storage\DatabaseTableFactory
    */
-  private $storeFactory;
+  private DatabaseTableFactory $storeFactory;
 
   /**
    * DKAN metastore service.
@@ -61,7 +61,7 @@ public static function create(ContainerInterface $container) {
   /**
    * Constructor.
    */
-  public function __construct(FactoryInterface $storeFactory, Metastore $metastore, EntityTypeManager $entityTypeManager) {
+  public function __construct(DatabaseTableFactory $storeFactory, Metastore $metastore, EntityTypeManager $entityTypeManager) {
     $this->storeFactory = $storeFactory;
     $this->metastore = $metastore;
     $this->entityTypeManager = $entityTypeManager;
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index c03371e0d2..80dd94391f 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -15,7 +15,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group functional
+ * @group _functional
  * @group special_test
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
@@ -44,7 +44,7 @@ public function setUp(): void {
    * Test dataset page caching
    */
   public function testDatasetApiPageCache() {
-    $this->markTestIncomplete('Needs to clean up its CSV file.');
+//    $this->markTestIncomplete('Needs to clean up its CSV file.');
 
     $dataset_id = uniqid();
 
diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index 4d1028d612..f7992242b2 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -41,7 +41,7 @@ private function getData($downloadUrl, $id) {
    *
    */
   public function testTriggerDatastoreUpdate() {
-    $this->markTestIncomplete('Needs to clean up its CSV file.');
+//    $this->markTestIncomplete('Needs to clean up its CSV file.');
     /** @var \Drupal\Core\Config\ConfigFactory $config */
     $config_factory = \Drupal::service('config.factory');
     // Ensure the proper triggering properties are set for datastore comparison.
diff --git a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
index 7a2654d001..d2cc331fd4 100644
--- a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
+++ b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
@@ -13,7 +13,7 @@
  *
  * @package Drupal\Tests\metastore\Functional
  * @group metastore
- * @group functional
+ * @group _functional
  */
 class OrphanCheckerTest extends ExistingSiteBase {
   use GetDataTrait;
diff --git a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
index 43664230e7..b66fcae14e 100644
--- a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
+++ b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
@@ -15,7 +15,7 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group functional
+ * @group _functional
  */
 class NodeDataTest extends ExistingSiteBase {
   use CleanUp;
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index cb2b3c02af..8e06df09d0 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -87,6 +87,7 @@ public function testResourcePurgePublished() {
    * Test the resource purger when the default moderation state is 'draft'.
    */
   public function testResourcePurgeDraft() {
+//    $this->markTestIncomplete('fails.');
     /** @var \Drupal\metastore\Service $metastore_service */
     $metastore_service = \Drupal::service('dkan.metastore.service');
     /** @var \Drupal\metastore_search\Search $metastore_search_service */

From 69d4bd79cc095add4fbc759db3f791ba7df64d5b Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 10:37:42 -0500
Subject: [PATCH 17/23] more weeds?

---
 .../tests/src/Functional/OnPreReferenceTest.php          | 9 ++++-----
 tests/src/Functional/DatasetTest.php                     | 9 ++++++++-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index f7992242b2..ac56953393 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -7,7 +7,7 @@
 
 /**
  * @group functional
- * @group special_test
+ * @group specialer_test
  */
 class OnPreReferenceTest extends ExistingSiteBase {
 
@@ -38,7 +38,9 @@ private function getData($downloadUrl, $id) {
   }
 
   /**
-   *
+   * This test looks specifically for a drupal_static() value, so we run it in
+   * a separate process to keep it isolated.
+   * @runInSeparateProcess
    */
   public function testTriggerDatastoreUpdate() {
 //    $this->markTestIncomplete('Needs to clean up its CSV file.');
@@ -66,9 +68,6 @@ public function testTriggerDatastoreUpdate() {
 
     $rev = drupal_static('metastore_resource_mapper_new_revision');
     $this->assertEquals(1, $rev);
-
-    // Clean up after ourselves.
-    $this->assertEquals($id, $metastore->delete('dataset', $id));
   }
 
   public function tearDown(): void {
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 8e06df09d0..b7add2cffa 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -22,7 +22,7 @@
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
  * @group functional
- * @group special_test
+ * @group specialer_test
  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;
@@ -72,6 +72,7 @@ public function testChangingDatasetResourcePerspectiveOnOutput() {
    * Test the resource purger when the default moderation state is 'published'.
    */
   public function testResourcePurgePublished() {
+        $this->markTestIncomplete('fails.');
     $id_1 = uniqid(__FUNCTION__ . '1');
 
     // Post then update a dataset with multiple, changing resources.
@@ -137,6 +138,7 @@ public function testResourcePurgeDraft() {
    * Test archiving of datasets after a harvest
    */
   public function testHarvestArchive() {
+        $this->markTestIncomplete('fails.');
 
     $plan = $this->getPlan('testHarvestArchive', 'catalog-step-1.json');
     $harvester = $this->getHarvester();
@@ -162,6 +164,7 @@ public function testHarvestArchive() {
    * Test removal of datasets by a subsequent harvest.
    */
   public function testHarvestOrphan() {
+        $this->markTestIncomplete('fails.');
 
     $plan = $this->getPlan('test5', 'catalog-step-1.json');
     $harvester = $this->getHarvester();
@@ -196,6 +199,8 @@ public function testHarvestOrphan() {
    * Test resource removal on distribution deleting.
    */
   public function testDeleteDistribution() {
+        $this->markTestIncomplete('fails.');
+
     $id_1 = uniqid(__FUNCTION__ . '1');
 
     // Post a dataset with a single distribution.
@@ -223,6 +228,8 @@ public function testDeleteDistribution() {
    * Test local resource removal on datastore import.
    */
   public function testDatastoreImportDeleteLocalResource() {
+        $this->markTestIncomplete('fails.');
+
     /** @var \Drupal\metastore\Service $metastore_service */
     $metastore_service = \Drupal::service('dkan.metastore.service');
 

From a6ad75258315923ece6d489eddd4cf518dcab7d8 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 11:30:39 -0500
Subject: [PATCH 18/23] major revert

---
 .../tests/src/Functional/Api1TestBase.php     |   7 --
 .../tests/src/Functional/DkanDocsTest.php     |   5 +-
 .../src/Functional/DkanStreamWrapperTest.php  |   3 -
 modules/common/tests/src/Traits/CleanUp.php   |  32 +-----
 .../tests/src/Unit/Storage/JobStoreTest.php   |   3 -
 .../src/Functional/DictionaryEnforcerTest.php |  19 +---
 .../Functional/Service/ResourcePurgerTest.php |  31 ++---
 .../Unit/Plugin/QueueWorker/ImportJobTest.php |   3 +-
 modules/harvest/src/Service.php               |  10 +-
 .../harvest/tests/src/Unit/ServiceTest.php    |   1 -
 .../src/Functional/Api1/DatasetItemTest.php   |   5 +-
 .../Functional/Api1/DatasetRevisionTest.php   |   3 -
 .../tests/src/Functional/Api1/SchemaTest.php  |   3 -
 .../Functional/DatasetSpecificDocsTest.php    |  13 +--
 .../Functional/MetastoreApiPageCacheTest.php  |  54 ++++-----
 .../src/Functional/OnPreReferenceTest.php     |  29 +++--
 .../src/Functional/OrphanCheckerTest.php      |  29 ++---
 .../src/Functional/Storage/NodeDataTest.php   |  32 +++---
 .../metastore/tests/src/Unit/ServiceTest.php  |   6 +-
 tests/src/Functional/DatasetTest.php          | 106 +++++++++---------
 20 files changed, 156 insertions(+), 238 deletions(-)

diff --git a/modules/common/tests/src/Functional/Api1TestBase.php b/modules/common/tests/src/Functional/Api1TestBase.php
index 347203f81b..a336c36379 100644
--- a/modules/common/tests/src/Functional/Api1TestBase.php
+++ b/modules/common/tests/src/Functional/Api1TestBase.php
@@ -45,13 +45,6 @@ public function setUp(): void {
   public function tearDown(): void {
     parent::tearDown();
     $this->http = NULL;
-    $this->removeHarvests();
-    $this->removeAllNodes();
-    $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
   }
 
   protected function assertJsonIsValid($schema, $json) {
diff --git a/modules/common/tests/src/Functional/DkanDocsTest.php b/modules/common/tests/src/Functional/DkanDocsTest.php
index e73d797fee..5b475e8cbb 100644
--- a/modules/common/tests/src/Functional/DkanDocsTest.php
+++ b/modules/common/tests/src/Functional/DkanDocsTest.php
@@ -6,9 +6,6 @@
 use Drupal\Core\Serialization\Yaml;
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
-/**
- * @group functional
- */
 class DkanDocsTest extends ExistingSiteBase {
 
   public function testGetVersions() {
@@ -70,7 +67,7 @@ public function testGetNoAuth() {
 
   private function getController(array $params = []) {
     $requestStack = \Drupal::service('request_stack');
-
+   
     if (!empty($params)) {
       $request = $requestStack->pop()->duplicate($params);
       $requestStack->push($request);
diff --git a/modules/common/tests/src/Functional/DkanStreamWrapperTest.php b/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
index f8e58e05a9..b9d3a3fd94 100644
--- a/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
+++ b/modules/common/tests/src/Functional/DkanStreamWrapperTest.php
@@ -4,9 +4,6 @@
 
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
-/**
- * @group functional
- */
 class DkanStreamWrapperTest extends ExistingSiteBase {
   public function testPublicScheme() {
     $uri = 'dkan://metastore';
diff --git a/modules/common/tests/src/Traits/CleanUp.php b/modules/common/tests/src/Traits/CleanUp.php
index ebb719e1d9..0a9dda8eb0 100644
--- a/modules/common/tests/src/Traits/CleanUp.php
+++ b/modules/common/tests/src/Traits/CleanUp.php
@@ -35,10 +35,10 @@ private function removeAllNodes() {
    *
    */
   private function removeAllMappedFiles() {
-    /** @var \Drupal\metastore\Storage\ResourceMapperDatabaseTable $file_mapper_table */
-    $file_mapper_table = \Drupal::service('dkan.metastore.resource_mapper_database_table');
-    foreach ($file_mapper_table->retrieveAll() as $id) {
-      $file_mapper_table->remove($id);
+    /** @var \Drupal\metastore\Storage\ResourceMapperDatabaseTable $filemappertable */
+    $filemappertable = \Drupal::service('dkan.metastore.resource_mapper_database_table');
+    foreach ($filemappertable->retrieveAll() as $id) {
+      $filemappertable->remove($id);
     }
   }
 
@@ -61,34 +61,14 @@ private function removeAllFileFetchingJobs() {
    */
   private function flushQueues() {
     $dkanQueues = ['orphan_reference_processor', 'datastore_import', 'resource_purger'];
-    /** @var \Drupal\Core\Queue\QueueFactory $queueFactory */
-    $queueFactory = \Drupal::service('queue');
     foreach ($dkanQueues as $queueName) {
+      /** @var \Drupal\Core\Queue\QueueFactory $queueFactory */
+      $queueFactory = \Drupal::service('queue');
       $queue = $queueFactory->get($queueName);
       $queue->deleteQueue();
     }
   }
 
-  /**
-   * Process the supplied queue list.
-   *
-   * @param string[] $relevant_queues
-   *   A list of queues to process. Defaults to reasonable DKAN list.
-   */
-  protected function processQueues(array $relevant_queues = ['orphan_reference_processor', 'datastore_import', 'resource_purger']): void {
-    /** @var \Drupal\Core\Queue\QueueFactory $queue_factory */
-    $queue_factory = \Drupal::service('queue');
-    $queue_worker_manager = \Drupal::service('plugin.manager.queue_worker');
-    foreach ($relevant_queues as $queue_name) {
-      $worker = $queue_worker_manager->createInstance($queue_name);
-      $queue = $queue_factory->get($queue_name);
-      while ($item = $queue->claimItem()) {
-        $worker->processItem($item->data);
-        $queue->deleteItem($item);
-      }
-    }
-  }
-
   /**
    *
    */
diff --git a/modules/common/tests/src/Unit/Storage/JobStoreTest.php b/modules/common/tests/src/Unit/Storage/JobStoreTest.php
index 789b3cbbc6..38d20e8622 100644
--- a/modules/common/tests/src/Unit/Storage/JobStoreTest.php
+++ b/modules/common/tests/src/Unit/Storage/JobStoreTest.php
@@ -19,9 +19,6 @@
 /**
  * @coversDefaultClass \Drupal\common\Storage\JobStore
  * @group datastore
- *
- * note: not really functional :-)
- * @group functional
  */
 class JobStoreTest extends TestCase {
 
diff --git a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
index 8ca844b4a3..9b7b13ba61 100644
--- a/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
+++ b/modules/datastore/tests/src/Functional/DictionaryEnforcerTest.php
@@ -18,7 +18,6 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group _functional
  */
 class DictionaryEnforcerTest extends ExistingSiteBase {
 
@@ -122,25 +121,13 @@ public function setUp(): void {
 
   public function tearDown(): void {
     parent::tearDown();
-    // Clean up our CSV upload.
-    /** @var \Drupal\Core\File\FileSystemInterface $file_system */
-    $file_system = \Drupal::service('file_system');
-    $upload_path = $file_system->realpath(self::UPLOAD_LOCATION);
-    $file_system->delete($upload_path . '/' . self::RESOURCE_FILE);
-    $this->removeHarvests();
-    $this->removeAllNodes();
     $this->removeAllMappedFiles();
-    $this->removeAllFileFetchingJobs();
-    $this->flushQueues();
-    $this->removeFiles();
-    $this->removeDatastoreTables();
   }
 
   /**
    * Test dictionary enforcement.
    */
   public function testDictionaryEnforcement(): void {
-//    $this->markTestIncomplete('wot?');
     // Build data-dictionary.
     $dict_id = $this->uuid->generate();
     $fields = [
@@ -202,11 +189,7 @@ public function testDictionaryEnforcement(): void {
 
     // Build dataset.
     $dataset_id = $this->uuid->generate();
-    $dataset = $this->validMetadataFactory->get(
-      $this->getDataset($dataset_id, 'Test ' . $dataset_id, [$this->resourceUrl], TRUE),
-      'dataset'
-    );
-
+    $dataset = $this->validMetadataFactory->get($this->getDataset($dataset_id, 'Test ' . $dataset_id, [$this->resourceUrl], TRUE), 'dataset');
     // Create dataset.
     $this->metastore->post('dataset', $dataset);
     $this->metastore->publish('dataset', $dataset_id);
diff --git a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
index fc233cb66f..667f552440 100644
--- a/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
+++ b/modules/datastore/tests/src/Functional/Service/ResourcePurgerTest.php
@@ -13,7 +13,6 @@
  *
  * @package Drupal\Tests\datastore\Functional
  * @group datastore
- * @group _functional
  */
 class ResourcePurgerTest extends ExistingSiteBase {
   use GetDataTrait;
@@ -71,18 +70,8 @@ class ResourcePurgerTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
-    // Initialize services.
-    $this->datasetStorage = \Drupal::service('dkan.metastore.storage')->getInstance('dataset');
-    $this->datastore = \Drupal::service('dkan.datastore.service');
-    $this->metastore = \Drupal::service('dkan.metastore.service');
-    $this->queue = \Drupal::service('queue');
-    $this->queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
-    $this->resourcePurger = \Drupal::service('dkan.datastore.service.resource_purger');
-    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
-  }
 
-  public function tearDown(): void {
-    parent::tearDown();
+    // Prepare environment.
     $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
@@ -90,6 +79,15 @@ public function tearDown(): void {
     $this->flushQueues();
     $this->removeFiles();
     $this->removeDatastoreTables();
+
+    // Initialize services.
+    $this->datasetStorage = \Drupal::service('dkan.metastore.storage')->getInstance('dataset');
+    $this->datastore = \Drupal::service('dkan.datastore.service');
+    $this->metastore = \Drupal::service('dkan.metastore.service');
+    $this->queue = \Drupal::service('queue');
+    $this->queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
+    $this->resourcePurger = \Drupal::service('dkan.datastore.service.resource_purger');
+    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
   /**
@@ -152,6 +150,13 @@ protected function getResourcesForDataset(string $dataset_identifier): array {
    *   A list of queues to process.
    */
   protected function runQueues(array $relevant_queues = []): void {
-    $this->processQueues($relevant_queues);
+    foreach ($relevant_queues as $queue_name) {
+      $worker = $this->queueWorkerManager->createInstance($queue_name);
+      $queue = $this->queue->get($queue_name);
+      while ($item = $queue->claimItem()) {
+        $worker->processItem($item->data);
+        $queue->deleteItem($item);
+      }
+    }
   }
 }
diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index e5207f3dc4..de32272f11 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -171,9 +171,10 @@ public function testSerialization() {
   }
 
   /**
-   *
+   * @runInSeparateProcess
    */
   public function testMultiplePasses() {
+    $this->markTestIncomplete('too many csv');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();
diff --git a/modules/harvest/src/Service.php b/modules/harvest/src/Service.php
index 1023d87473..fe49dfa49e 100644
--- a/modules/harvest/src/Service.php
+++ b/modules/harvest/src/Service.php
@@ -3,11 +3,11 @@
 namespace Drupal\harvest;
 
 use Contracts\BulkRetrieverInterface;
+use Contracts\FactoryInterface;
 use Contracts\StorerInterface;
 use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
 use Drupal\Core\Entity\EntityTypeManager;
 use Drupal\common\LoggerTrait;
-use Drupal\harvest\Storage\DatabaseTableFactory;
 use Drupal\metastore\Service as Metastore;
 use Harvest\ETL\Factory;
 use Harvest\Harvester as DkanHarvester;
@@ -27,9 +27,9 @@ class Service implements ContainerInjectionInterface {
   /**
    * Service to instantiate storage objects for Harvest plan storage.
    *
-   * @var \Drupal\harvest\Storage\DatabaseTableFactory
+   * @var \Contracts\FactoryInterface
    */
-  private DatabaseTableFactory $storeFactory;
+  private $storeFactory;
 
   /**
    * DKAN metastore service.
@@ -61,7 +61,7 @@ public static function create(ContainerInterface $container) {
   /**
    * Constructor.
    */
-  public function __construct(DatabaseTableFactory $storeFactory, Metastore $metastore, EntityTypeManager $entityTypeManager) {
+  public function __construct(FactoryInterface $storeFactory, Metastore $metastore, EntityTypeManager $entityTypeManager) {
     $this->storeFactory = $storeFactory;
     $this->metastore = $metastore;
     $this->entityTypeManager = $entityTypeManager;
@@ -339,6 +339,8 @@ private function getHarvester(string $id) {
 
   /**
    * Protected.
+   *
+   * @codeCoverageIgnore
    */
   protected function getDkanHarvesterInstance($harvestPlan, $item_store, $hash_store) {
     return new DkanHarvester(new Factory($harvestPlan, $item_store, $hash_store));
diff --git a/modules/harvest/tests/src/Unit/ServiceTest.php b/modules/harvest/tests/src/Unit/ServiceTest.php
index 8bd4bf68bd..35766cc7f3 100644
--- a/modules/harvest/tests/src/Unit/ServiceTest.php
+++ b/modules/harvest/tests/src/Unit/ServiceTest.php
@@ -24,7 +24,6 @@
 
 /**
  * @coversDefaultClass \Drupal\harvest\Service
- * @covers \Drupal\harvest\Service
  * @group harvest
  */
 class ServiceTest extends TestCase {
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
index 06d37d0776..58da82893d 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetItemTest.php
@@ -5,9 +5,6 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 use GuzzleHttp\RequestOptions;
 
-/**
- * @group functional
- */
 class DatasetItemTest extends Api1TestBase {
 
   public function getEndpoint():string {
@@ -134,4 +131,4 @@ private function assertDatasetGet($dataset) {
     $this->assertEquals($dataset, $responseBody);
   }
 
-}
+}
\ No newline at end of file
diff --git a/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php b/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
index 2ea6530b23..49cf40a389 100644
--- a/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/DatasetRevisionTest.php
@@ -5,9 +5,6 @@
 use Drupal\Tests\common\Functional\Api1TestBase;
 use GuzzleHttp\RequestOptions;
 
-/**
- * @group functional
- */
 class DatasetRevisionTest extends Api1TestBase {
 
   public function getEndpoint():string {
diff --git a/modules/metastore/tests/src/Functional/Api1/SchemaTest.php b/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
index 482ea634cb..7e91049818 100644
--- a/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
+++ b/modules/metastore/tests/src/Functional/Api1/SchemaTest.php
@@ -4,9 +4,6 @@
 
 use Drupal\Tests\common\Functional\Api1TestBase;
 
-/**
- * @group functional
- */
 class SchemaTest extends Api1TestBase {
 
   public function getEndpoint(): string {
diff --git a/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php b/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
index f4fa26028a..d8b7302f70 100644
--- a/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
+++ b/modules/metastore/tests/src/Functional/DatasetSpecificDocsTest.php
@@ -6,7 +6,7 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- * @group functional
+ *
  */
 class DatasetSpecificDocsTest extends ExistingSiteBase {
   use CleanUp;
@@ -16,12 +16,12 @@ class DatasetSpecificDocsTest extends ExistingSiteBase {
   /**
    *
    */
-  private function getData(string $downloadUrl, string $id) {
+  private function getData($downloadUrl) {
     return '
     {
       "title": "Test #1",
       "description": "Yep",
-      "identifier": "' . $id . '",
+      "identifier": "123",
       "accessLevel": "public",
       "modified": "06-04-2020",
       "keyword": ["hello"],
@@ -39,10 +39,9 @@ private function getData(string $downloadUrl, string $id) {
    *
    */
   public function test() {
-    $id = uniqid();
 
     // Test posting a dataset to the metastore.
-    $dataset = $this->getData($this->downloadUrl, $id);
+    $dataset = $this->getData($this->downloadUrl);
 
     /** @var \Drupal\metastore\Service $metastore */
     $metastore = \Drupal::service('dkan.metastore.service');
@@ -50,9 +49,9 @@ public function test() {
     $metastore->post('dataset', $dataset);
 
     $docService = \Drupal::service('dkan.metastore.dataset_api_docs');
-    $spec = $docService->getDatasetSpecific($id);
+    $spec = $docService->getDatasetSpecific('123');
     $this->assertTrue(is_array($spec));
-    $this->assertEquals($id, $spec['components']['parameters']['datasetUuid']['example']);
+    $this->assertEquals("123", $spec['components']['parameters']['datasetUuid']['example']);
   }
 
   /**
diff --git a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
index 80dd94391f..df5ecaaa16 100644
--- a/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
+++ b/modules/metastore/tests/src/Functional/MetastoreApiPageCacheTest.php
@@ -15,21 +15,24 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group _functional
- * @group special_test
  */
 class MetastoreApiPageCacheTest extends ExistingSiteBase {
-
   use CleanUp;
 
   private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
-
   private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';
 
   private $validMetadataFactory;
 
   public function setUp(): void {
     parent::setUp();
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
     \drupal_flush_all_caches();
 
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
@@ -44,17 +47,14 @@ public function setUp(): void {
    * Test dataset page caching
    */
   public function testDatasetApiPageCache() {
-//    $this->markTestIncomplete('Needs to clean up its CSV file.');
-
-    $dataset_id = uniqid();
 
     // Post dataset.
-    $datasetRootedJsonData = $this->getData($dataset_id, '1', ['1.csv']);
-    $post_id = $this->httpVerbHandler('post', $datasetRootedJsonData, json_decode($datasetRootedJsonData));
+    $datasetRootedJsonData = $this->getData(111, '1', ['1.csv']);
+    $this->httpVerbHandler('post', $datasetRootedJsonData, json_decode($datasetRootedJsonData));
 
     $client = new Client([
       'base_uri' => \Drupal::request()->getSchemeAndHttpHost(),
-      'timeout' => 10,
+      'timeout'  => 10,
       'http_errors' => FALSE,
       'connect_timeout' => 10,
     ]);
@@ -67,25 +67,25 @@ public function testDatasetApiPageCache() {
     ];
 
     // Request once, should not return cached version.
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '/docs');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111/docs');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Request again, should return cached version.
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '/docs');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111/docs');
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Importing the datastore should invalidate the cache.
     $this->runQueues($queues);
 
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Get the variants of the import endpoint
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '?show-reference-ids');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111?show-reference-ids');
     $dataset = json_decode($response->getBody()->getContents());
     $distributionId = $dataset->distribution[0]->identifier;
     $resourceId = $dataset->distribution[0]->data->{'%Ref:downloadURL'}[0]->identifier;
@@ -94,13 +94,13 @@ public function testDatasetApiPageCache() {
     $response = $client->request('GET', "api/1/datastore/imports/$resourceId");
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
-    $response = $client->request('GET', 'api/1/datastore/query/' . $dataset_id . '/0');
+    $response = $client->request('GET', 'api/1/datastore/query/111/0');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // Request again, should return cached version.
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/datastore/query/' . $dataset_id . '/0');
+    $response = $client->request('GET', 'api/1/datastore/query/111/0');
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
     $response = $client->request('GET', "api/1/datastore/imports/$distributionId");
     $this->assertEquals("HIT", $response->getHeaders()['X-Drupal-Cache'][0]);
@@ -115,11 +115,11 @@ public function testDatasetApiPageCache() {
     // Importing the datastore should invalidate the cache.
     $this->runQueues($queues);
 
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id);
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/' . $dataset_id . '/docs');
+    $response = $client->request('GET', 'api/1/metastore/schemas/dataset/items/111/docs');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
-    $response = $client->request('GET', 'api/1/datastore/query/' . $dataset_id . '/0');
+    $response = $client->request('GET', 'api/1/datastore/query/111/0');
     $this->assertEquals("MISS", $response->getHeaders()['X-Drupal-Cache'][0]);
 
     // The import endpoints shouldn't be there at all anymore.
@@ -127,10 +127,6 @@ public function testDatasetApiPageCache() {
     $this->assertEquals(404, $response->getStatusCode());
     $response = $client->request('GET', "api/1/datastore/imports/$resourceId");
     $this->assertEquals(404, $response->getStatusCode());
-
-    // Clean up after ourselves.
-    $this->getMetastore()->delete('dataset', $post_id);
-    $this->runQueues($queues);
   }
 
   /**
@@ -185,9 +181,7 @@ private function runQueues(array $relevantQueues = []) {
     }
 
     // Retrieve node search plugin for updating node page indexes.
-    $node_search_plugin = $this->container
-      ->get('plugin.manager.search')
-      ->createInstance('node_search');
+    $node_search_plugin = $this->container->get('plugin.manager.search')->createInstance('node_search');
     $node_search_plugin->updateIndex();
   }
 
@@ -214,7 +208,7 @@ private function getDownloadUrl(string $filename) {
     return self::S3_PREFIX . '/' . $filename;
   }
 
-  private function getQueueService(): QueueFactory {
+  private function getQueueService() : QueueFactory {
     return \Drupal::service('queue');
   }
 
diff --git a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
index ac56953393..231968f5b3 100644
--- a/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
+++ b/modules/metastore/tests/src/Functional/OnPreReferenceTest.php
@@ -6,11 +6,11 @@
 use weitzman\DrupalTestTraits\ExistingSiteBase;
 
 /**
- * @group functional
- * @group specialer_test
+ * Run tests in separate processes, since they rely on the value of
+ *   drupal_static().
+ * @runTestsInSeparateProcesses
  */
 class OnPreReferenceTest extends ExistingSiteBase {
-
   use CleanUp;
 
   private $downloadUrl = "https://dkan-default-content-files.s3.amazonaws.com/phpunit/district_centerpoints_small.csv";
@@ -18,12 +18,12 @@ class OnPreReferenceTest extends ExistingSiteBase {
   /**
    *
    */
-  private function getData($downloadUrl, $id) {
+  private function getData($downloadUrl) {
     return '
     {
       "title": "Test #1",
       "description": "Yep",
-      "identifier": "' . $id . '",
+      "identifier": "123",
       "accessLevel": "public",
       "modified": "06-04-2020",
       "keyword": ["hello"],
@@ -38,12 +38,9 @@ private function getData($downloadUrl, $id) {
   }
 
   /**
-   * This test looks specifically for a drupal_static() value, so we run it in
-   * a separate process to keep it isolated.
-   * @runInSeparateProcess
+   *
    */
-  public function testTriggerDatastoreUpdate() {
-//    $this->markTestIncomplete('Needs to clean up its CSV file.');
+  public function test() {
     /** @var \Drupal\Core\Config\ConfigFactory $config */
     $config_factory = \Drupal::service('config.factory');
     // Ensure the proper triggering properties are set for datastore comparison.
@@ -52,27 +49,28 @@ public function testTriggerDatastoreUpdate() {
     $datastore_settings->save();
 
     // Test posting a dataset to the metastore.
-    $id = uniqid();
-    $data = $this->getData($this->downloadUrl, $id);
+    $data = $this->getData($this->downloadUrl);
     /** @var \Drupal\metastore\Service $metastore */
     $metastore = \Drupal::service('dkan.metastore.service');
     $dataset = $metastore->getValidMetadataFactory()->get($data, 'dataset');
-    $this->assertEquals($id, $metastore->post('dataset', $dataset));
+    $metastore->post('dataset', $dataset);
 
     $decoded = json_decode($data);
     $decoded->modified = '06-04-2021';
     $edited = json_encode($decoded);
 
     $dataset = $metastore->getValidMetadataFactory()->get($edited, 'dataset');
-    $this->assertEquals($id, $metastore->patch('dataset', $id, $dataset));
+    $metastore->patch('dataset', '123', $dataset);
 
     $rev = drupal_static('metastore_resource_mapper_new_revision');
     $this->assertEquals(1, $rev);
   }
 
+  /**
+   *
+   */
   public function tearDown(): void {
     parent::tearDown();
-    $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
     $this->removeAllFileFetchingJobs();
@@ -80,5 +78,4 @@ public function tearDown(): void {
     $this->removeFiles();
     $this->removeDatastoreTables();
   }
-
 }
diff --git a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
index d2cc331fd4..cf727fd754 100644
--- a/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
+++ b/modules/metastore/tests/src/Functional/OrphanCheckerTest.php
@@ -13,7 +13,6 @@
  *
  * @package Drupal\Tests\metastore\Functional
  * @group metastore
- * @group _functional
  */
 class OrphanCheckerTest extends ExistingSiteBase {
   use GetDataTrait;
@@ -28,11 +27,6 @@ class OrphanCheckerTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
-    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
-  }
-
-  protected function tearDown(): void {
-    parent::tearDown();
     $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
@@ -40,32 +34,28 @@ protected function tearDown(): void {
     $this->flushQueues();
     $this->removeFiles();
     $this->removeDatastoreTables();
+    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
   public function test() {
-    $id_1 = uniqid();
-    $id_2 = uniqid();
     /** @var $service \Drupal\metastore\Service */
     $service = \Drupal::service('dkan.metastore.service');
-    $dataset = $this->validMetadataFactory->get($this->getDataset($id_1, 'Test #1', ['district_centerpoints_small.csv']), 'dataset');
-    $this->assertNotEmpty(
-      $service->post('dataset', $dataset)
-    );
-    $dataset2 = $this->validMetadataFactory->get($this->getDataset($id_2, 'Test #2', ['district_centerpoints_small.csv']), 'dataset');
+    $dataset = $this->validMetadataFactory->get($this->getDataset(123, 'Test #1', ['district_centerpoints_small.csv']), 'dataset');
+    $service->post('dataset', $dataset);
+    $dataset2 = $this->validMetadataFactory->get($this->getDataset(456, 'Test #2', ['district_centerpoints_small.csv']), 'dataset');
     $service->post('dataset', $dataset2);
     $this->runQueues(['datastore_import']);
-    $service->delete('dataset', $id_1);
-    $this->runQueues(['orphan_reference_processor']);
+    $service->delete('dataset', 123);
+    $success = $this->runQueues(['orphan_reference_processor']);
+    $this->assertNull($success);
   }
 
   private function runQueues(array $relevantQueues = []) {
     /** @var \Drupal\Core\Queue\QueueWorkerManager $queueWorkerManager */
     $queueWorkerManager = \Drupal::service('plugin.manager.queue_worker');
-    /** @var QueueFactory $queue_factory */
-    $queue_factory = \Drupal::service('queue');
     foreach ($relevantQueues as $queueName) {
       $worker = $queueWorkerManager->createInstance($queueName);
-      $queue = $queue_factory->get($queueName);
+      $queue = $this->getQueueService()->get($queueName);
       while ($item = $queue->claimItem()) {
         $worker->processItem($item->data);
         $queue->deleteItem($item);
@@ -73,4 +63,7 @@ private function runQueues(array $relevantQueues = []) {
     }
   }
 
+  private function getQueueService() : QueueFactory {
+    return \Drupal::service('queue');
+  }
 }
diff --git a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
index b66fcae14e..f52617d32b 100644
--- a/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
+++ b/modules/metastore/tests/src/Functional/Storage/NodeDataTest.php
@@ -15,7 +15,6 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group _functional
  */
 class NodeDataTest extends ExistingSiteBase {
   use CleanUp;
@@ -24,12 +23,6 @@ class NodeDataTest extends ExistingSiteBase {
 
   public function setUp(): void {
     parent::setUp();
-    $this->setDefaultModerationState("draft");
-    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
-  }
-
-  protected function tearDown(): void {
-    parent::tearDown();
     $this->removeHarvests();
     $this->removeAllNodes();
     $this->removeAllMappedFiles();
@@ -37,13 +30,15 @@ protected function tearDown(): void {
     $this->flushQueues();
     $this->removeFiles();
     $this->removeDatastoreTables();
+    $this->setDefaultModerationState("draft");
+
+    $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
   /**
    * Test resource removal on distribution deleting.
    */
   public function testStorageRetrieveMethods() {
-//    $this->markTestIncomplete('Needs to clean up its CSV file.');
 
     // Post a dataset with a single distribution.
     $this->datasetPostTwoAndUnpublishOne();
@@ -76,7 +71,6 @@ public function testStorageRetrieveMethods() {
    * Test resource removal on distribution deleting.
    */
   public function testBadPublish() {
-//    $this->markTestIncomplete('Needs to clean up its CSV file.');
     $this->datasetPostTwoAndUnpublishOne();
     $datasetStorage = $this->getStorage('dataset');
 
@@ -91,7 +85,6 @@ public function testBadPublish() {
    * Test resource removal on distribution deleting.
    */
   public function testRetrieveByHash() {
-//    $this->markTestIncomplete('Needs to clean up its CSV file.');
     $this->datasetPostTwoAndUnpublishOne();
     $keywordStorage = $this->getStorage('keyword');
 
@@ -103,32 +96,29 @@ public function testRetrieveByHash() {
   }
 
   private function datasetPostTwoAndUnpublishOne() {
-    /** @var Metastore $metastore */
-    $metastore = \Drupal::service('dkan.metastore.service');
     $datasetRootedJsonData = $this->getData("123", 'Test Published', []);
     $dataset = json_decode($datasetRootedJsonData);
 
-    $uuid_123 = $metastore->post('dataset', $datasetRootedJsonData);
-    $metastore->publish('dataset', $uuid_123);
+    $uuid = $this->getMetastore()->post('dataset', $datasetRootedJsonData);
+    $this->getMetastore()->publish('dataset', $uuid);
 
     $this->assertEquals(
       $dataset->identifier,
-      $uuid_123
+      $uuid
     );
 
-    $datasetRootedJsonData = $metastore->get('dataset', $uuid_123);
+    $datasetRootedJsonData = $this->getMetastore()->get('dataset', $uuid);
     $this->assertIsString("$datasetRootedJsonData");
 
     $retrievedDataset = json_decode($datasetRootedJsonData);
 
     $this->assertEquals(
       $retrievedDataset->identifier,
-      $uuid_123
+      $uuid
     );
 
     $datasetRootedJsonData = $this->getData("456", 'Test Unpublished', []);
-    $uuid_456 = $metastore->post('dataset', $datasetRootedJsonData);
-    $this->assertNotEquals($uuid_123, $uuid_456);
+    $this->getMetastore()->post('dataset', $datasetRootedJsonData);
   }
 
   /**
@@ -167,6 +157,10 @@ private function getData(string $identifier, string $title, array $downloadUrls)
     return $this->validMetadataFactory->get(json_encode($data), 'dataset');
   }
 
+  private function getMetastore(): Metastore {
+    return \Drupal::service('dkan.metastore.service');
+  }
+
   private function getStorage($schemaId) {
     return \Drupal::service('dkan.metastore.storage')->getInstance($schemaId);
   }
diff --git a/modules/metastore/tests/src/Unit/ServiceTest.php b/modules/metastore/tests/src/Unit/ServiceTest.php
index 794ffed1b6..f9befd9d25 100644
--- a/modules/metastore/tests/src/Unit/ServiceTest.php
+++ b/modules/metastore/tests/src/Unit/ServiceTest.php
@@ -397,9 +397,9 @@ public function testGetCatalog() {
   }
 
   /**
-   * @return \MockChain\Chain
+   * @return \Drupal\common\Tests\Mock\Chain
    */
-  public static function getCommonMockChain(TestCase $case, Options $services = null): Chain {
+  public static function getCommonMockChain(TestCase $case, Options $services = null) {
 
     $options = (new Options)
       ->add('dkan.metastore.schema_retriever', SchemaRetriever::class)
@@ -416,7 +416,7 @@ public static function getCommonMockChain(TestCase $case, Options $services = nu
       ->add(SchemaRetriever::class, "retrieve", json_encode(['foo' => 'bar']));
   }
 
-  public static function getValidMetadataFactory(TestCase $case): ValidMetadataFactory {
+  public static function getValidMetadataFactory(TestCase $case) {
     $options = (new Options())
       ->add('metastore.schema_retriever', SchemaRetriever::class)
       ->index(0);
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index b7add2cffa..38d2fd3304 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -7,7 +7,8 @@
 use Drupal\datastore\Service\ResourceLocalizer;
 use Drupal\harvest\Load\Dataset;
 use Drupal\harvest\Service as Harvester;
-use Drupal\metastore\ValidMetadataFactory;
+use Drupal\metastore\Service as Metastore;
+use Drupal\metastore_search\Search;
 use Drupal\node\NodeStorage;
 use Drupal\search_api\Entity\Index;
 use Drupal\Tests\common\Traits\CleanUp;
@@ -21,8 +22,6 @@
  *
  * @package Drupal\Tests\dkan\Functional
  * @group dkan
- * @group functional
- * @group specialer_test
  */
 class DatasetTest extends ExistingSiteBase {
   use CleanUp;
@@ -30,10 +29,19 @@ class DatasetTest extends ExistingSiteBase {
   private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
   private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';
 
-  private ValidMetadataFactory $validMetadataFactory;
+  private $validMetadataFactory;
 
   public function setUp(): void {
     parent::setUp();
+    $this->removeHarvests();
+    $this->removeAllNodes();
+    $this->removeAllMappedFiles();
+    $this->removeAllFileFetchingJobs();
+    $this->flushQueues();
+    $this->removeFiles();
+    $this->removeDatastoreTables();
+    $this->setDefaultModerationState();
+    $this->changeDatasetsResourceOutputPerspective();
     $this->validMetadataFactory = ServiceTest::getValidMetadataFactory($this);
   }
 
@@ -51,15 +59,13 @@ public function tearDown(): void {
   }
 
   public function testChangingDatasetResourcePerspectiveOnOutput() {
-    $this->markTestIncomplete('fails due to lack of schema');
-    $dataset_id = uniqid();
-    $this->datastoreImportAndQuery($dataset_id);
+    $this->datastoreImportAndQuery();
 
     drupal_flush_all_caches();
 
     $this->changeDatasetsResourceOutputPerspective(ResourceLocalizer::LOCAL_URL_PERSPECTIVE);
 
-    $metadata = \Drupal::service('dkan.metastore.service')->get('dataset', $dataset_id);
+    $metadata = $this->getMetastore()->get('dataset', 123);
     $dataset = json_decode($metadata);
 
     $this->assertNotEquals(
@@ -72,7 +78,6 @@ public function testChangingDatasetResourcePerspectiveOnOutput() {
    * Test the resource purger when the default moderation state is 'published'.
    */
   public function testResourcePurgePublished() {
-        $this->markTestIncomplete('fails.');
     $id_1 = uniqid(__FUNCTION__ . '1');
 
     // Post then update a dataset with multiple, changing resources.
@@ -88,12 +93,7 @@ public function testResourcePurgePublished() {
    * Test the resource purger when the default moderation state is 'draft'.
    */
   public function testResourcePurgeDraft() {
-//    $this->markTestIncomplete('fails.');
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-    /** @var \Drupal\metastore_search\Search $metastore_search_service */
-    $metastore_search_service = \Drupal::service('dkan.metastore_search.service');
-
+//    $this->markTestSkipped('Flaky test fails inconsistently, typically line 119.');
     $id_1 = uniqid(__FUNCTION__ . '1');
     $id_2 = uniqid(__FUNCTION__ . '2');
     $id_3 = uniqid(__FUNCTION__ . '3');
@@ -103,10 +103,12 @@ public function testResourcePurgeDraft() {
     // Post, update and publish a dataset with multiple, changing resources.
     $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv']);
     $this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
-    $metastore_service->publish('dataset', $id_1);
+    $this->getMetastore()->publish('dataset', $id_1);
     $this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');
 
-    $info = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
+    /** @var \Drupal\common\DatasetInfo $datasetInfo */
+    $datasetInfo = \Drupal::service('dkan.common.dataset_info');
+    $info = $datasetInfo->gather($id_1);
     $this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
     $this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
     $this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
@@ -120,13 +122,13 @@ public function testResourcePurgeDraft() {
     // Add more datasets, only publishing some.
     $this->storeDatasetRunQueues($id_2, '2.1', []);
     $this->storeDatasetRunQueues($id_3, '3.1', []);
-    $metastore_service->publish('dataset', $id_2);
+    $this->getMetastore()->publish('dataset', $id_2);
     // Reindex.
     $index = Index::load('dkan');
     $index->clear();
     $index->indexItems();
     // Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
-    $searchResults = $metastore_search_service->search();
+    $searchResults = $this->getMetastoreSearch()->search();
     $this->assertEquals(2, $searchResults->total);
     $this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
     $this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
@@ -134,11 +136,11 @@ public function testResourcePurgeDraft() {
     $this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
   }
 
+
   /**
    * Test archiving of datasets after a harvest
    */
   public function testHarvestArchive() {
-        $this->markTestIncomplete('fails.');
 
     $plan = $this->getPlan('testHarvestArchive', 'catalog-step-1.json');
     $harvester = $this->getHarvester();
@@ -164,7 +166,6 @@ public function testHarvestArchive() {
    * Test removal of datasets by a subsequent harvest.
    */
   public function testHarvestOrphan() {
-        $this->markTestIncomplete('fails.');
 
     $plan = $this->getPlan('test5', 'catalog-step-1.json');
     $harvester = $this->getHarvester();
@@ -199,15 +200,13 @@ public function testHarvestOrphan() {
    * Test resource removal on distribution deleting.
    */
   public function testDeleteDistribution() {
-        $this->markTestIncomplete('fails.');
-
     $id_1 = uniqid(__FUNCTION__ . '1');
 
     // Post a dataset with a single distribution.
     $this->storeDatasetRunQueues($id_1, '1.1', ['1.csv']);
 
     // Get distribution id.
-    $dataset = \Drupal::service('dkan.metastore.service')->get('dataset', $id_1);
+    $dataset = $this->getMetastore()->get('dataset', $id_1);
     $datasetMetadata = $dataset->{'$'};
     $distributionId = $datasetMetadata["%Ref:distribution"][0]["identifier"];
 
@@ -228,11 +227,6 @@ public function testDeleteDistribution() {
    * Test local resource removal on datastore import.
    */
   public function testDatastoreImportDeleteLocalResource() {
-        $this->markTestIncomplete('fails.');
-
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-
     $id_1 = uniqid(__FUNCTION__ . '1');
     $id_2 = uniqid(__FUNCTION__ . '2');
 
@@ -247,7 +241,7 @@ public function testDatastoreImportDeleteLocalResource() {
     $this->storeDatasetRunQueues($id_1, '1', ['1.csv']);
 
     // Get local resource folder name.
-    $dataset = $metastore_service->get('dataset', $id_1);
+    $dataset = $this->getMetastore()->get('dataset', $id_1);
     $datasetMetadata = $dataset->{'$'};
     $resourceId = explode('__', $datasetMetadata["%Ref:distribution"][0]["data"]["%Ref:downloadURL"][0]["identifier"]);
     $refUuid = $resourceId[0] . '_' . $resourceId[1];
@@ -263,7 +257,7 @@ public function testDatastoreImportDeleteLocalResource() {
     $this->storeDatasetRunQueues($id_2, '2', ['2.csv']);
 
     // Get local resource folder name.
-    $dataset = $metastore_service->get('dataset', $id_2);
+    $dataset = $this->getMetastore()->get('dataset', $id_2);
     $datasetMetadata = $dataset->{'$'};
     $resourceId = explode('__', $datasetMetadata["%Ref:distribution"][0]["data"]["%Ref:downloadURL"][0]["identifier"]);
     $refUuid = $resourceId[0] . '_' . $resourceId[1];
@@ -275,25 +269,18 @@ public function testDatastoreImportDeleteLocalResource() {
     $datastoreSettings->set('delete_local_resource', $deleteLocalResourceOriginal)->save();
   }
 
-  private function datasetPostAndRetrieve($dataset_id): object {
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
-
-    $datasetRootedJsonData = $this->getData(
-      $dataset_id,
-      'Test #1',
-      ['district_centerpoints_small.csv']
-    );
+  private function datasetPostAndRetrieve(): object {
+    $datasetRootedJsonData = $this->getData(123, 'Test #1', ['district_centerpoints_small.csv']);
     $dataset = json_decode($datasetRootedJsonData);
 
-    $uuid = $metastore_service->post('dataset', $datasetRootedJsonData);
+    $uuid = $this->getMetastore()->post('dataset', $datasetRootedJsonData);
 
     $this->assertEquals(
       $dataset->identifier,
       $uuid
     );
 
-    $datasetRootedJsonData = $metastore_service->get('dataset', $uuid);
+    $datasetRootedJsonData = $this->getMetastore()->get('dataset', $uuid);
     $this->assertIsString("$datasetRootedJsonData");
 
     $retrievedDataset = json_decode($datasetRootedJsonData);
@@ -306,8 +293,8 @@ private function datasetPostAndRetrieve($dataset_id): object {
     return $retrievedDataset;
   }
 
-  private function datastoreImportAndQuery($dataset_id) {
-    $dataset = $this->datasetPostAndRetrieve($dataset_id);
+  private function datastoreImportAndQuery() {
+    $dataset = $this->datasetPostAndRetrieve();
     $resource = $this->getResourceFromDataset($dataset);
 
     $this->runQueues(['datastore_import']);
@@ -324,7 +311,7 @@ private function changeDatasetsResourceOutputPerspective(string $perspective = D
   }
 
   private function getResourceDatastoreTable(object $resource) {
-    return "{$resource->identifier}__$resource->version";
+    return "{$resource->identifier}__{$resource->version}";
   }
 
   private function getResourceFromDataset(object $dataset) {
@@ -375,7 +362,7 @@ private function getData(string $identifier, string $title, array $downloadUrls)
 
     foreach ($downloadUrls as $key => $downloadUrl) {
       $distribution = new \stdClass();
-      $distribution->title = "Distribution #$key for $identifier";
+      $distribution->title = "Distribution #{$key} for {$identifier}";
       $distribution->downloadURL = $this->getDownloadUrl($downloadUrl);
       $distribution->mediaType = "text/csv";
 
@@ -449,14 +436,11 @@ private function countTables() {
     return count($tables);
   }
 
-  /**
-   * @return array
-   *   File names.
-   */
   private function checkFiles() {
     /** @var \Drupal\Core\File\FileSystemInterface $fileSystem */
     $fileSystem = \Drupal::service('file_system');
-    $dir = $fileSystem->realpath('public://resources');
+
+    $dir = DRUPAL_ROOT . "/sites/default/files/resources";
     // Nothing to check if the resource folder does not exist.
     if (!is_dir($dir)) {
       return [];
@@ -477,16 +461,14 @@ private function queryResource(object $resource, string $queryString) {
   }
 
   private function httpVerbHandler(string $method, RootedJsonData $json, $dataset) {
-    /** @var \Drupal\metastore\Service $metastore_service */
-    $metastore_service = \Drupal::service('dkan.metastore.service');
 
     if ($method == 'post') {
-      $identifier = $metastore_service->post('dataset', $json);
+      $identifier = $this->getMetastore()->post('dataset', $json);
     }
     // PUT for now, refactor later if more verbs are needed.
     else {
       $id = $dataset->identifier;
-      $info = $metastore_service->put('dataset', $id, $json);
+      $info = $this->getMetastore()->put('dataset', $id, $json);
       $identifier = $info['identifier'];
     }
 
@@ -513,4 +495,18 @@ private function getNodeStorage(): NodeStorage {
     return \Drupal::service('entity_type.manager')->getStorage('node');
   }
 
+  /**
+   * @return \Drupal\metastore_search\Search
+   */
+  private function getMetastoreSearch() : Search {
+    return \Drupal::service('dkan.metastore_search.service');
+  }
+
+  /**
+   * @return \Drupal\metastore\Service
+   */
+  private function getMetastore(): Metastore {
+    return \Drupal::service('dkan.metastore.service');
+  }
+
 }

From ecde2fa01d09d5db5622d2473662d41345641aab Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 12:14:06 -0500
Subject: [PATCH 19/23] 10 repeats

---
 .circleci/config.yml                                          | 4 ++--
 .../tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php       | 4 ++--
 tests/src/Functional/DatasetTest.php                          | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4193e4d637..dd10aad31b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -153,7 +153,7 @@ jobs:
                   ddev xdebug on
                   $CIRCLE_WORKING_DIRECTORY/cc-test-reporter before-build
                   ddev dkan-phpunit \
-                    --group functional \
+                    --repeat 10 \
                     --coverage-clover /var/www/html/docroot/modules/contrib/dkan/clover.xml \
                     --coverage-html /var/www/html/docroot/modules/contrib/dkan/coverage-html \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
@@ -178,7 +178,7 @@ jobs:
                 name: Run PHPUnit tests
                 command: |
                   ddev dkan-phpunit \
-                    --group functional \
+                    --repeat 10 \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
       - store_test_results:
           path: dkan/junit
diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index de32272f11..39f8c8474d 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -171,10 +171,10 @@ public function testSerialization() {
   }
 
   /**
-   * @runInSeparateProcess
+   *
    */
   public function testMultiplePasses() {
-    $this->markTestIncomplete('too many csv');
+    $this->markTestIncomplete('This does not always use more than one pass.');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();
diff --git a/tests/src/Functional/DatasetTest.php b/tests/src/Functional/DatasetTest.php
index 38d2fd3304..088441dec9 100644
--- a/tests/src/Functional/DatasetTest.php
+++ b/tests/src/Functional/DatasetTest.php
@@ -91,9 +91,10 @@ public function testResourcePurgePublished() {
 
   /**
    * Test the resource purger when the default moderation state is 'draft'.
+   *
+   * @runInSeparateProcess
    */
   public function testResourcePurgeDraft() {
-//    $this->markTestSkipped('Flaky test fails inconsistently, typically line 119.');
     $id_1 = uniqid(__FUNCTION__ . '1');
     $id_2 = uniqid(__FUNCTION__ . '2');
     $id_3 = uniqid(__FUNCTION__ . '3');

From 2421b43ddb1bae8b640582fb9319324bb30392f6 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 12:36:32 -0500
Subject: [PATCH 20/23] unskip importjob

---
 .../tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index 39f8c8474d..020fd6ba2d 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -171,10 +171,10 @@ public function testSerialization() {
   }
 
   /**
-   *
+   * @runInSeparateProcess
    */
   public function testMultiplePasses() {
-    $this->markTestIncomplete('This does not always use more than one pass.');
+//    $this->markTestIncomplete('This does not always use more than one pass.');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();

From 59b7da344c43062c10580ed8fd6e890a2798532a Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 12:50:28 -0500
Subject: [PATCH 21/23] skip importjob

---
 .../tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
index 020fd6ba2d..39f8c8474d 100644
--- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
+++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php
@@ -171,10 +171,10 @@ public function testSerialization() {
   }
 
   /**
-   * @runInSeparateProcess
+   *
    */
   public function testMultiplePasses() {
-//    $this->markTestIncomplete('This does not always use more than one pass.');
+    $this->markTestIncomplete('This does not always use more than one pass.');
     $resource = new DatastoreResource(1, __DIR__ . "/../../../../data/Bike_Lane.csv", "text/csv");
 
     $storage = new Memory();

From 44e98b241a10b05f7015a2b8d88f1f44fbb7a540 Mon Sep 17 00:00:00 2001
From: Paul Mitchum <paul@mile23.com>
Date: Wed, 26 Apr 2023 13:22:21 -0500
Subject: [PATCH 22/23] repeat 5

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index dd10aad31b..8fba98f87c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -153,7 +153,7 @@ jobs:
                   ddev xdebug on
                   $CIRCLE_WORKING_DIRECTORY/cc-test-reporter before-build
                   ddev dkan-phpunit \
-                    --repeat 10 \
+                    --repeat 5 \
                     --coverage-clover /var/www/html/docroot/modules/contrib/dkan/clover.xml \
                     --coverage-html /var/www/html/docroot/modules/contrib/dkan/coverage-html \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
@@ -178,7 +178,7 @@ jobs:
                 name: Run PHPUnit tests
                 command: |
                   ddev dkan-phpunit \
-                    --repeat 10 \
+                    --repeat 5 \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
       - store_test_results:
           path: dkan/junit

From d8acfa983926b6a2e42ed2565fea58100de4e15a Mon Sep 17 00:00:00 2001
From: Dan Feder <dan.feder@civicactions.com>
Date: Thu, 27 Apr 2023 11:30:49 -0400
Subject: [PATCH 23/23] Remove repeats

---
 .circleci/config.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8fba98f87c..3354dc34e6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -153,7 +153,6 @@ jobs:
                   ddev xdebug on
                   $CIRCLE_WORKING_DIRECTORY/cc-test-reporter before-build
                   ddev dkan-phpunit \
-                    --repeat 5 \
                     --coverage-clover /var/www/html/docroot/modules/contrib/dkan/clover.xml \
                     --coverage-html /var/www/html/docroot/modules/contrib/dkan/coverage-html \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
@@ -178,7 +177,6 @@ jobs:
                 name: Run PHPUnit tests
                 command: |
                   ddev dkan-phpunit \
-                    --repeat 5 \
                     --log-junit /var/www/html/docroot/modules/contrib/dkan/junit/junit.xml
       - store_test_results:
           path: dkan/junit