Use original headers on csv downloads

modules/datastore/src/Controller/AbstractQueryController.php

@@ -3,15 +3,15 @@
 namespace Drupal\datastore\Controller;
 
 use Drupal\common\DatasetInfo;
-use Drupal\datastore\Service\DatastoreQuery;
-use Drupal\datastore\Service\Query as QueryService;
-use Symfony\Component\DependencyInjection\ContainerInterface;
-use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
 use Drupal\common\JsonResponseTrait;
-use RootedData\RootedJsonData;
 use Drupal\Core\Config\ConfigFactoryInterface;
+use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
+use Drupal\datastore\Service\DatastoreQuery;
+use Drupal\datastore\Service\Query as QueryService;
 use Drupal\metastore\MetastoreApiResponse;
 use JsonSchema\Validator;
+use RootedData\RootedJsonData;
+use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\HttpFoundation\ParameterBag;
 use Symfony\Component\HttpFoundation\Request;
 
@@ -170,9 +170,9 @@ public function queryDatasetResource(string $dataset, string $index, Request $re
    *
    * Abstract method; override in specific implementations.
    *
-   * @param Drupal\datastore\Service\DatastoreQuery $datastoreQuery
+   * @param \Drupal\datastore\Service\DatastoreQuery $datastoreQuery
    *   A datastore query object.
-   * @param RootedData\RootedJsonData $result
+   * @param \RootedData\RootedJsonData $result
    *   The result of the datastore query.
    * @param array $dependencies
    *   A dependency array for use by \Drupal\metastore\MetastoreApiResponse.
@@ -226,6 +226,10 @@ protected function buildDatastoreQuery(Request $request, $identifier = NULL) {
       $resource = (object) ["id" => $identifier, "alias" => "t"];
       $data->resources = [$resource];
     }
+    // Force schema if CSV.
+    if (($data->format ?? NULL) == 'csv') {
+      $data->schema = TRUE;
+    }
     return new DatastoreQuery(json_encode($data), $this->getRowsLimit());
   }
 
@@ -344,4 +348,57 @@ public static function fixTypes($json, $schema) {
     return json_encode($data, JSON_PRETTY_PRINT);
   }
 
+  /**
+   * Build a CSV header row based on a query and result.
+   *
+   * @param \Drupal\datastore\Service\DatastoreQuery $datastoreQuery
+   *   A datastore query object.
+   * @param \RootedData\RootedJsonData $result
+   *   The result of the datastore query.
+   *
+   * @return array
+   *   Array of strings for a CSV header row.
+   */
+  protected function getHeaderRow(DatastoreQuery $datastoreQuery, RootedJsonData &$result) {
+    $schema_fields = $result->{'$.schema..fields'}[0] ?? [];
+    if (empty($schema_fields)) {
+      throw new \DomainException("Could not generate header for CSV.");
+    }
+    if (empty($datastoreQuery->{'$.properties'})) {
+      return array_keys($schema_fields);
+    }
+
+    $header_row = [];
+    foreach ($datastoreQuery->{'$.properties'} ?? [] as $property) {
+      $normalized_prop = $this->propToString($property, $datastoreQuery);
+      $header_row[] = $schema_fields[$normalized_prop]['description'] ?? $normalized_prop;
+    }
+
+    return $header_row;
+  }
+
+  /**
+   * Transform any property into a string for mapping to schema.
+   *
+   * @param string|array $property
+   *   A property from a DataStore Query.
+   *
+   * @return string
+   *   String version of property.
+   */
+  protected function propToString(string|array $property): string {
+    if (is_string($property)) {
+      return $property;
+    }
+    elseif (isset($property['property'])) {
+      return $property['property'];
+    }
+    elseif (isset($property['alias'])) {
+      return $property['alias'];
+    }
+    else {
+      throw new \DomainException("Invalid property: " . print_r($property, TRUE));
+    }
+  }
+
 }

modules/datastore/src/Controller/QueryController.php

@@ -3,8 +3,8 @@
 namespace Drupal\datastore\Controller;
 
 use Drupal\datastore\Service\DatastoreQuery;
-use RootedData\RootedJsonData;
 use Ilbee\CSVResponse\CSVResponse;
+use RootedData\RootedJsonData;
 use Symfony\Component\HttpFoundation\ParameterBag;
 
 /**
@@ -17,9 +17,9 @@ class QueryController extends AbstractQueryController {
   /**
    * Return correct JSON or CSV response.
    *
-   * @param Drupal\datastore\Service\DatastoreQuery $datastoreQuery
+   * @param \Drupal\datastore\Service\DatastoreQuery $datastoreQuery
    *   A datastore query object.
-   * @param RootedData\RootedJsonData $result
+   * @param \RootedData\RootedJsonData $result
    *   The result of the datastore query.
    * @param array $dependencies
    *   A dependency array for use by \Drupal\metastore\MetastoreApiResponse.
@@ -37,7 +37,8 @@ public function formatResponse(
   ) {
     switch ($datastoreQuery->{"$.format"}) {
       case 'csv':
-        $response = new CSVResponse($result->{"$.results"}, 'data.csv', ',');
+        $results = $this->useCsvHeaders($datastoreQuery, $result);
+        $response = new CSVResponse($results, 'data.csv', ',');
         return $this->addCacheHeaders($response);
 
       case 'json':
@@ -46,6 +47,28 @@ public function formatResponse(
     }
   }
 
+  /**
+   * Use csv column names based on specified properties or the schema.
+   *
+   * Alters the data array.
+   *
+   * @param \Drupal\datastore\Service\DatastoreQuery $datastoreQuery
+   *   A datastore query object.
+   * @param \RootedData\RootedJsonData $result
+   *   The result of the datastore query.
+   */
+  private function useCsvHeaders(DatastoreQuery $datastoreQuery, RootedJsonData &$result) {
+    $header_row = $this->getHeaderRow($datastoreQuery, $result);
+    $rows = $result->{"$.results"};
+    $newResults = [];
+    $newRows = [];
+    foreach ($rows as $row) {
+      $newRows = array_combine($header_row, array_values($row));
+      array_push($newResults, $newRows);
+    }
+    return $newResults;
+  }
+
   /**
    * Retrieve the datastore query schema. Used by datastore.1.query.schema.get.
    *

modules/datastore/src/Controller/QueryDownloadController.php

@@ -3,9 +3,9 @@
 namespace Drupal\datastore\Controller;
 
 use Drupal\datastore\Service\DatastoreQuery;
-use Symfony\Component\HttpFoundation\StreamedResponse;
 use RootedData\RootedJsonData;
 use Symfony\Component\HttpFoundation\ParameterBag;
+use Symfony\Component\HttpFoundation\StreamedResponse;
 
 /**
  * Controller providing functionality used to stream datastore queries.
@@ -73,7 +73,7 @@ protected function streamCsvResponse(DatastoreQuery $datastoreQuery, RootedJsonD
       // Wrap in try/catch so that we can still close the output buffer.
       try {
         // Send the header row.
-        $this->sendRow($handle, $this->getHeaderRow($result));
+        $this->sendRow($handle, $this->getHeaderRow($datastoreQuery, $result));
 
         // Get the result pointer and send each row to the stream one by one.
         $result = $this->queryService->runResultsQuery($datastoreQuery, FALSE, TRUE);
@@ -118,39 +118,4 @@ private function sendRow($handle, array $row) {
     flush();
   }
 
-  /**
-   * Add the header row from specified properties or the schema.
-   *
-   * Alters the data array.
-   *
-   * @param \RootedData\RootedJsonData $result
-   *   The result of a DatastoreQuery.
-   */
-  private function getHeaderRow(RootedJsonData &$result) {
-
-    try {
-      if (!empty($result->{'$.query.properties'})) {
-        $header_row = $result->{'$.query.properties'};
-      }
-      else {
-        $schema = $result->{'$.schema'};
-        // Query has are no explicit properties; we should assume one table.
-        $header_row = array_keys(reset($schema)['fields']);
-      }
-      if (empty($header_row) || !is_array($header_row)) {
-        throw new \DomainException("Could not generate header for CSV.");
-      }
-    }
-    catch (\Exception $e) {
-      throw new \DomainException("Could not generate header for CSV.");
-    }
-
-    array_walk($header_row, function (&$header) {
-      if (is_array($header)) {
-        $header = $header['alias'] ?? $header['property'];
-      }
-    });
-    return $header_row;
-  }
-
 }

modules/datastore/src/Service/Query.php

@@ -3,11 +3,11 @@
 namespace Drupal\datastore\Service;
 
 use Drupal\common\DataResource;
-use RootedData\RootedJsonData;
-use Symfony\Component\DependencyInjection\ContainerInterface;
 use Drupal\Core\DependencyInjection\ContainerInjectionInterface;
 use Drupal\datastore\DatastoreService;
 use Drupal\datastore\Storage\QueryFactory;
+use RootedData\RootedJsonData;
+use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * Datastore query service.
@@ -51,6 +51,8 @@ public function __construct(DatastoreService $datastore) {
    */
   public function runQuery(DatastoreQuery $datastoreQuery) {
     $return = (object) [];
+
+    $this->getProperties($datastoreQuery);
     if ($datastoreQuery->{"$.results"} !== FALSE) {
       $return->results = $this->runResultsQuery($datastoreQuery);
     }
@@ -132,12 +134,6 @@ public function runResultsQuery(DatastoreQuery $datastoreQuery, $fetch = TRUE, $
     }
     $storageMap = $this->getQueryStorageMap($datastoreQuery);
 
-    $storage = $storageMap[$primaryAlias];
-
-    if (empty($datastoreQuery->{"$.rowIds"}) && empty($datastoreQuery->{"$.properties"}) && $storage->getSchema()) {
-      $schema = $this->filterSchemaFields($storage->getSchema(), $storage->primaryKey());
-      $datastoreQuery->{"$.properties"} = array_keys($schema['fields']);
-    }
     $query = QueryFactory::create($datastoreQuery, $storageMap);
     // Get data dictionary fields.
     $meta_data = $csv != FALSE ? $this->getDatastoreService()->getDataDictionaryFields() : NULL;
@@ -220,4 +216,25 @@ private function runCountQuery(DatastoreQuery $datastoreQuery) {
     return (int) $storageMap[$primaryAlias]->query($query, $primaryAlias)[0]->expression;
   }
 
+  /**
+   * Under most circumstances, we want an explicit list of properties.
+   *
+   * @param \Drupal\datastore\Service\DatastoreQuery $datastoreQuery
+   *   Datastore query object to be modified.
+   */
+  private function getProperties(DatastoreQuery $datastoreQuery) {
+    $primaryAlias = $datastoreQuery->{"$.resources[0].alias"};
+    if (!$primaryAlias) {
+      return [];
+    }
+    $storageMap = $this->getQueryStorageMap($datastoreQuery);
+
+    $storage = $storageMap[$primaryAlias];
+
+    if (!($datastoreQuery->{"$.rowIds"} ?? FALSE) && empty($datastoreQuery->{"$.properties"}) && $storage->getSchema()) {
+      $schema = $this->filterSchemaFields($storage->getSchema(), $storage->primaryKey());
+      $datastoreQuery->{"$.properties"} = array_keys($schema['fields'] ?? []);
+    }
+  }
+
 }

modules/datastore/tests/src/Unit/Controller/QueryControllerTest.php

@@ -35,7 +35,7 @@ class QueryControllerTest extends TestCase {
 
   protected function setUp(): void {
     parent::setUp();
-    // Set cache services
+    // Set cache services.
     $options = (new Options)
       ->add('cache_contexts_manager', CacheContextsManager::class)
       ->index(0);
@@ -269,8 +269,8 @@ public function testResourceQueryJoins() {
             "value" => [
               "resource" => "t",
               "property" => "record_number",
-            ]
-          ]
+            ],
+          ],
         ],
       ],
     ]);
@@ -301,13 +301,13 @@ public function testQueryCsv() {
     $this->assertEquals(200, $result->getStatusCode());
 
     $csv = explode("\n", $result->getContent());
-    $this->assertEquals('state,year', $csv[0]);
+    $this->assertEquals('State,Year', $csv[0]);
     $this->assertEquals('Alabama,2010', $csv[1]);
     $this->assertStringContainsString('data.csv', $result->headers->get('Content-Disposition'));
   }
 
   private function getQueryResult($data, $id = NULL, $index = NULL, $info = []) {
-    $container = $this->getQueryContainer($data, $info)->getMock();
+    $container = $this->getQueryContainer($data, $info, true)->getMock();
     $webServiceApi = QueryController::create($container);
     $request = $this->mockRequest($data);
     if ($id === NULL && $index === NULL) {
@@ -319,17 +319,51 @@ private function getQueryResult($data, $id = NULL, $index = NULL, $info = []) {
     return $webServiceApi->queryDatasetResource($id, $index, $request);
   }
 
-  /**
-   *
-   */
   public function testResourceQueryCsv() {
     $data = json_encode([
+      "properties" => [
+        [
+          "resource" => "t",
+          "property" => "state",
+        ],
+      ],
       "results" => TRUE,
       "format" => "csv",
     ]);
-    $result = $this->getQueryResult($data);
+    $result = $this->getQueryResult($data, "2");
+    $this->assertTrue($result instanceof CsvResponse);
+    $csv = explode("\n", $result->getContent());
+    $this->assertEquals('State', $csv[0]);
+    $this->assertEquals('Alabama', $csv[1]);
+    $this->assertEquals(200, $result->getStatusCode());
+  }
+
 
+  public function testResourceExpressionQueryCsv() {
+    $data = json_encode([
+      "properties" => [
+        "state",
+        "year",
+        [
+          "expression" => [
+            "operator" => "+",
+            "operands" => [
+              "year",
+              1,
+            ],
+          ],
+          "alias" => "year_plus_1",
+        ],
+      ],
+      "results" => TRUE,
+      "format" => "csv",
+    ]);
+    $result = $this->getQueryResult($data, "2");
     $this->assertTrue($result instanceof CsvResponse);
+
+    $csv = explode("\n", $result->getContent());
+    $this->assertEquals('State,Year,year_plus_1', $csv[0]);
+    $this->assertEquals('Alabama,2010,2011', $csv[1]);
     $this->assertEquals(200, $result->getStatusCode());
   }
 
@@ -508,9 +542,9 @@ public function mockDatastoreTable() {
     );
     $storage->setSchema([
       'fields' => [
-        'record_number' => ['type' => 'int', 'not null' => TRUE],
-        'state' => ['type' => 'text'],
-        'year' => ['type' => 'int'],
+        'record_number' => ['type' => 'int', 'description' => 'Record Number', 'not null' => TRUE],
+        'state' => ['type' => 'text', 'description' => 'State'],
+        'year' => ['type' => 'int', 'description' => 'Year'],
       ],
     ]);
     return $storage;