HARMONY-1105: Fix issue with concatenation not being called for the L…

…2-subsetter-to-concise chain.
nasa · Mar 15, 2022 · bc4eaaa · bc4eaaa
1 parent 1a017ee
commit bc4eaaa
Show file tree

Hide file tree

Showing 7 changed files with 198 additions and 1 deletion.
diff --git a/app/models/services/base-service.ts b/app/models/services/base-service.ts
@@ -95,7 +95,7 @@ const conditionToOperationField = {
   variableSubset: 'shouldVariableSubset',
   spatialSubset: 'shouldSpatialSubset',
   temporalSubset: 'shouldTemporalSubset',
-  concatenate: 'concatenate',
+  concatenate: 'shouldConcatenate',
 };
 
 /**

diff --git a/fixtures/cmr.uat.earthdata.nasa.gov-443/164737495950661052 b/fixtures/cmr.uat.earthdata.nasa.gov-443/164737495950661052
@@ -0,0 +1,29 @@
+POST /search/granules.json
+accept: application/json
+content-type: multipart/form-data; boundary=----------------------------012345678901234567890123
+accept-encoding: gzip,deflate
+body: ------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"collection_concept_id\"\r\n\r\nC1234208438-POCLOUD\r\n------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"page_size\"\r\n\r\n3\r\n------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"scroll\"\r\n\r\ndefer\r\n------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"bounding_box\"\r\n\r\n-180,0,180,90\r\n------------------------------012345678901234567890123--\r\n
+
+HTTP/1.1 200 OK
+content-type: application/json;charset=utf-8
+content-length: 157
+connection: close
+date: Tue, 15 Mar 2022 20:09:19 GMT
+x-frame-options: SAMEORIGIN
+access-control-allow-origin: *
+x-xss-protection: 1; mode=block
+cmr-request-id: e55950ae-c954-4d57-9e7b-ab1fbbaf0329
+strict-transport-security: max-age=31536000
+cmr-hits: 7975
+access-control-expose-headers: CMR-Hits, CMR-Request-Id, X-Request-Id, CMR-Scroll-Id, CMR-Search-After, CMR-Timed-Out, CMR-Shapefile-Original-Point-Count, CMR-Shapefile-Simplified-Point-Count
+x-content-type-options: nosniff
+cmr-took: 84
+x-request-id: _5eVr229qAuKZqoOlPSpWkDoPdJvzzDmLFZtKhafikVLJcWWr5d9Aw==
+cmr-scroll-id: 2033507703
+server: ServerTokens ProductOnly
+x-cache: Miss from cloudfront
+via: 1.1 ec2068c87c2d80519a5753dcf9eaede6.cloudfront.net (CloudFront)
+x-amz-cf-pop: IAD66-C2
+x-amz-cf-id: _5eVr229qAuKZqoOlPSpWkDoPdJvzzDmLFZtKhafikVLJcWWr5d9Aw==
+
+{"feed":{"updated":"2022-03-15T20:09:19.485Z","id":"https://cmr.uat.earthdata.nasa.gov:443/search/granules.json","title":"ECHO granule metadata","entry":[]}}
diff --git a/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497527079659 b/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497527079659
diff --git a/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497570127654 b/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497570127654
diff --git a/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497597984679 b/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497597984679
@@ -0,0 +1,30 @@
+POST /search/granules.json
+accept: application/json
+content-type: multipart/form-data; boundary=----------------------------012345678901234567890123
+accept-encoding: gzip,deflate
+body: ------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"collection_concept_id\"\r\n\r\nC1233800302-EEDTEST\r\n------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"page_size\"\r\n\r\n2\r\n------------------------------012345678901234567890123--\r\n
+
+HTTP/1.1 200 OK
+content-type: application/json;charset=utf-8
+transfer-encoding: chunked
+connection: close
+vary: Accept-Encoding
+date: Tue, 15 Mar 2022 20:09:35 GMT
+x-frame-options: SAMEORIGIN
+access-control-allow-origin: *
+x-xss-protection: 1; mode=block
+cmr-request-id: d9f6d74e-547d-44fc-89b2-8816c7240048
+strict-transport-security: max-age=31536000
+cmr-search-after: ["eedtest",1577844000000,1233800344]
+cmr-hits: 177
+access-control-expose-headers: CMR-Hits, CMR-Request-Id, X-Request-Id, CMR-Scroll-Id, CMR-Search-After, CMR-Timed-Out, CMR-Shapefile-Original-Point-Count, CMR-Shapefile-Simplified-Point-Count
+x-content-type-options: nosniff
+cmr-took: 42
+x-request-id: 08ilssGNnGZ8vgMegfNBtmWyVTCt6ohQvaBtF9lw7FJusCANSFdoRQ==
+server: ServerTokens ProductOnly
+x-cache: Miss from cloudfront
+via: 1.1 7bab52277d6272f1dda6e20e26aba08e.cloudfront.net (CloudFront)
+x-amz-cf-pop: IAD66-C2
+x-amz-cf-id: 08ilssGNnGZ8vgMegfNBtmWyVTCt6ohQvaBtF9lw7FJusCANSFdoRQ==
+
+{"feed":{"updated":"2022-03-15T20:09:35.969Z","id":"https://cmr.uat.earthdata.nasa.gov:443/search/granules.json","title":"ECHO granule metadata","entry":[{"producer_granule_id":"001_00_7f00ff_global","boxes":["-90 -180 90 180"],"time_start":"2020-01-01T00:00:00.000Z","updated":"2020-02-25T21:43:11.000Z","dataset_id":"Harmony Example Data","data_center":"EEDTEST","title":"001_00_7f00ff_global","coordinate_system":"CARTESIAN","day_night_flag":"BOTH","time_end":"2020-01-01T01:59:59.000Z","id":"G1233800343-EEDTEST","original_format":"ECHO10","granule_size":"1.3701868057250977","browse_flag":false,"collection_concept_id":"C1233800302-EEDTEST","online_access_flag":true,"links":[{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"application/x-netcdf4","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/nc/001_00_7f00ff_global.nc"},{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"image/tiff","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/tiff/001_00_7f00ff_global.tif"}]},{"producer_granule_id":"001_01_7f00ff_africa","boxes":["-34.9 -25.4 37.4 63.5"],"time_start":"2020-01-01T02:00:00.000Z","updated":"2020-02-25T21:43:11.000Z","dataset_id":"Harmony Example Data","data_center":"EEDTEST","title":"001_01_7f00ff_africa","coordinate_system":"CARTESIAN","day_night_flag":"BOTH","time_end":"2020-01-01T03:59:59.000Z","id":"G1233800344-EEDTEST","original_format":"ECHO10","granule_size":"0.29718685150146484","browse_flag":false,"collection_concept_id":"C1233800302-EEDTEST","online_access_flag":true,"links":[{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"application/x-netcdf4","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/nc/001_01_7f00ff_africa.nc"},{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"image/tiff","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/tiff/001_01_7f00ff_africa.tif"}]}]}}
diff --git a/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497644651079 b/fixtures/cmr.uat.earthdata.nasa.gov-443/164737497644651079
@@ -0,0 +1,30 @@
+POST /search/granules.json
+accept: application/json
+content-type: multipart/form-data; boundary=----------------------------012345678901234567890123
+accept-encoding: gzip,deflate
+body: ------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"collection_concept_id\"\r\n\r\nC1233800302-EEDTEST\r\n------------------------------012345678901234567890123\r\nContent-Disposition: form-data; name=\"page_size\"\r\n\r\n3\r\n------------------------------012345678901234567890123--\r\n
+
+HTTP/1.1 200 OK
+content-type: application/json;charset=utf-8
+transfer-encoding: chunked
+connection: close
+vary: Accept-Encoding
+date: Tue, 15 Mar 2022 20:09:36 GMT
+x-frame-options: SAMEORIGIN
+access-control-allow-origin: *
+x-xss-protection: 1; mode=block
+cmr-request-id: 4642c41d-b806-4bd4-9962-ae8457132ef5
+strict-transport-security: max-age=31536000
+cmr-search-after: ["eedtest",1577844000000,1234866411]
+cmr-hits: 177
+access-control-expose-headers: CMR-Hits, CMR-Request-Id, X-Request-Id, CMR-Scroll-Id, CMR-Search-After, CMR-Timed-Out, CMR-Shapefile-Original-Point-Count, CMR-Shapefile-Simplified-Point-Count
+x-content-type-options: nosniff
+cmr-took: 45
+x-request-id: 0s__JTkAxDAKM52o3ulKZFghjnB1mBLssklVH6JDpQScJKN-qi034A==
+server: ServerTokens ProductOnly
+x-cache: Miss from cloudfront
+via: 1.1 603961b1b2dad8873023f00c2744bd6e.cloudfront.net (CloudFront)
+x-amz-cf-pop: IAD66-C2
+x-amz-cf-id: 0s__JTkAxDAKM52o3ulKZFghjnB1mBLssklVH6JDpQScJKN-qi034A==
+
+{"feed":{"updated":"2022-03-15T20:09:36.227Z","id":"https://cmr.uat.earthdata.nasa.gov:443/search/granules.json","title":"ECHO granule metadata","entry":[{"producer_granule_id":"001_00_7f00ff_global","boxes":["-90 -180 90 180"],"time_start":"2020-01-01T00:00:00.000Z","updated":"2020-02-25T21:43:11.000Z","dataset_id":"Harmony Example Data","data_center":"EEDTEST","title":"001_00_7f00ff_global","coordinate_system":"CARTESIAN","day_night_flag":"BOTH","time_end":"2020-01-01T01:59:59.000Z","id":"G1233800343-EEDTEST","original_format":"ECHO10","granule_size":"1.3701868057250977","browse_flag":false,"collection_concept_id":"C1233800302-EEDTEST","online_access_flag":true,"links":[{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"application/x-netcdf4","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/nc/001_00_7f00ff_global.nc"},{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"image/tiff","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/tiff/001_00_7f00ff_global.tif"}]},{"producer_granule_id":"001_01_7f00ff_africa","boxes":["-34.9 -25.4 37.4 63.5"],"time_start":"2020-01-01T02:00:00.000Z","updated":"2020-02-25T21:43:11.000Z","dataset_id":"Harmony Example Data","data_center":"EEDTEST","title":"001_01_7f00ff_africa","coordinate_system":"CARTESIAN","day_night_flag":"BOTH","time_end":"2020-01-01T03:59:59.000Z","id":"G1233800344-EEDTEST","original_format":"ECHO10","granule_size":"0.29718685150146484","browse_flag":false,"collection_concept_id":"C1233800302-EEDTEST","online_access_flag":true,"links":[{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"application/x-netcdf4","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/nc/001_01_7f00ff_africa.nc"},{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"image/tiff","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/tiff/001_01_7f00ff_africa.tif"}]},{"producer_granule_id":"001_01_7f00ff_africa_poly","time_start":"2020-01-01T02:00:00.000Z","updated":"2020-02-25T21:43:11.000Z","dataset_id":"Harmony Example Data","data_center":"EEDTEST","title":"001_01_7f00ff_africa_poly","coordinate_system":"CARTESIAN","day_night_flag":"BOTH","time_end":"2020-01-01T03:59:59.000Z","id":"G1234866411-EEDTEST","original_format":"ECHO10","granule_size":"0.29718685150146484","browse_flag":false,"polygons":[["36 5 34 25 34 63 37 63 37 25 36 5"]],"collection_concept_id":"C1233800302-EEDTEST","online_access_flag":true,"links":[{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"application/x-netcdf4","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/nc/001_01_7f00ff_africa.nc"},{"rel":"http://esipfed.org/ns/fedsearch/1.1/data#","type":"image/tiff","hreflang":"en-US","href":"https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/harmony_example/tiff/001_01_7f00ff_africa.tif"}]}]}}
diff --git a/test/concatenation.ts b/test/concatenation.ts
@@ -133,6 +133,56 @@ describe('testing concatenation', function () {
     });
   });
 
+  describe('for an L2 subsetter to CONCISE workflow', function () {
+    const collection = 'C1234208438-POCLOUD';
+    const l2SubsetterImage = 'ghcr.io/podaac/l2ss-py:sit';
+    const conciseImage = 'ghcr.io/podaac/concise:sit';
+
+    describe('When passing the concatenate parameter and spatial subsetting', function () {
+      hookServersStartStop( { skipEarthdataLogin: false });
+      const query = {
+        concatenate: true,
+        subset: 'lat(0:90)',
+        maxResults: 3,
+      };
+
+      describe('priming the test so it works... this is meaningless, but workflow steps are not created by later tests without it', function () {
+        StubService.hook({ params: { redirect: 'http://example.com' } });
+        hookRangesetRequest('1.0.0', collection, 'all', { query, username: 'joe' });
+      });
+
+      hookRangesetRequest('1.0.0', collection, 'all', { query, username: 'joe' });
+      hookRedirect('joe');
+
+      it('includes a workflow step to invoke the l2-subsetter service first', async function () {
+        const job = JSON.parse(this.res.text);
+        const workflowSteps = await getWorkflowStepsByJobId(db, job.jobID);
+        expect(workflowSteps[1].serviceID).to.eql(l2SubsetterImage);
+      });
+
+      it('does not set the `hasAggregatedOutput` flag on the l2-subsetter workflow step', async function () {
+        const job = JSON.parse(this.res.text);
+        const workflowSteps = await getWorkflowStepsByJobId(db, job.jobID);
+
+        expect(workflowSteps[1].hasAggregatedOutput).to.equal(0);
+      });
+
+      it('includes a workflow step to invoke the concise service', async function () {
+        const job = JSON.parse(this.res.text);
+        const workflowSteps = await getWorkflowStepsByJobId(db, job.jobID);
+
+        expect(workflowSteps[2].serviceID).to.eql(conciseImage);
+      });
+
+      it('has the `hasAggregatedOutput` flag set to true on the concise workflow step', async function () {
+        const job = JSON.parse(this.res.text);
+        const workflowSteps = await getWorkflowStepsByJobId(db, job.jobID);
+
+        expect(workflowSteps[2].hasAggregatedOutput).to.equal(1);
+      });
+    });
+  });
+
   describe('for a netcdf-to-zarr workflow', function () {
     const zarrCollection = 'C1233800302-EEDTEST';
     hookServersStartStop();