CUMULUS-3286:Handle get temporal info from non-existing CMR file (nas…

…a#3490) * CUMULUS-3286:Handle get temporal info from non-existing CMR file * update api test timeout * fix test * update granule es mapping queryFields * add unit test for s3ObjectExists
nsidc · Oct 9, 2023 · 9493d00 · 9493d00
1 parent 3e82063
commit 9493d00
Show file tree

Hide file tree

Showing 11 changed files with 123 additions and 38 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -64,6 +64,11 @@ Users/clients that do not make use of these endpoints will not be impacted.
 - **CUMULUS-3095**
   - Added back `rule` schema validation which is missing after RDS phase 3.
   - Fixed a bug for creating rule with tags.
+- **CUMULUS-3286**
+  - Fixed `@cumulus/cmrjs/cmr-utils/getGranuleTemporalInfo` and `@cumulus/message/Granules/getGranuleCmrTemporalInfo`
+    to handle non-existing cmr file.
+  - Updated mapping for granule and deletedgranule Elasticsearch records to prevent dynamic field for keys under
+    `queryFields`.
 - **CUMULUS-3393**
   - Fixed `PUT` collection endpoint to update collection configuration in S3.
 

diff --git a/example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js b/example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js
@@ -139,9 +139,11 @@ describe('The S3 Ingest Granules workflow', () => {
 
       // copiedToOrca contains a list of the file s3uri in primary buckets
       const copiedOver = await Promise.all(
-        filesCopiedToOrca.map((s3uri) => {
+        filesCopiedToOrca.map(async (s3uri) => {
           expect(excludedFileExtensions.filter((type) => s3uri.endsWith(type)).length).toBe(0);
-          return s3ObjectExists({ Bucket: config.buckets.glacier.name, Key: parseS3Uri(s3uri).Key });
+          const parsedS3Uri = parseS3Uri(s3uri);
+          await deleteS3Object(parsedS3Uri.Bucket, parsedS3Uri.Key);
+          return s3ObjectExists({ Bucket: config.buckets.glacier.name, Key: parsedS3Uri.Key });
         })
       );
       copiedOver.forEach((check) => expect(check).toEqual(true));

diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js
@@ -2059,6 +2059,9 @@ test.serial('PATCH updates an existing granule in all data stores', async (t) =>
     testExecutionCumulusId,
   } = t.context;
   const timestamp = Date.now();
+  const oldQueryFields = {
+    foo: Math.random(),
+  };
   const { newPgGranule, esRecord } = await createGranuleAndFiles({
     dbClient: knex,
     esClient,
@@ -2067,6 +2070,7 @@ test.serial('PATCH updates an existing granule in all data stores', async (t) =>
       status: 'running',
       execution: executionUrl,
       timestamp: Date.now(),
+      queryFields: oldQueryFields,
     },
   });
   const newApiGranule = await translatePostgresGranuleToApiGranule({
@@ -2075,9 +2079,9 @@ test.serial('PATCH updates an existing granule in all data stores', async (t) =>
   });
 
   t.is(newPgGranule.status, 'running');
-  t.is(newPgGranule.query_fields, null);
+  t.deepEqual(newPgGranule.query_fields, oldQueryFields);
   t.is(esRecord.status, 'running');
-  t.is(esRecord.queryFields, undefined);
+  t.deepEqual(esRecord.queryFields, oldQueryFields);
 
   const newQueryFields = {
     foo: randomString(),
@@ -3562,7 +3566,6 @@ test.serial('PUT replaces an existing granule in all data stores, removing exist
     executionUrl,
     knex,
   } = t.context;
-  console.log('foobar');
   const {
     apiGranule,
     newPgGranule,

diff --git a/packages/api/tests/lib/writeRecords/test-write-granules.js b/packages/api/tests/lib/writeRecords/test-write-granules.js
@@ -646,20 +646,20 @@ test.serial('writeGranulesFromMessage() on re-write saves granule records to Pos
   ];
 
   const completeGranule = fakeGranuleFactoryV2({
-    beginningDateTime: new Date().toString(),
+    beginningDateTime: new Date().toISOString(),
     cmrLink: 'example.com',
     collectionId: constructCollectionId(collection.name, collection.version),
     createdAt: Date.now(),
     duration: 1000,
-    endingDateTime: new Date().toString(),
+    endingDateTime: new Date().toISOString(),
     error: { errorKey: 'errorValue' },
     execution: executionUrl,
     files: files,
-    lastUpdateDateTime: new Date().toString(),
+    lastUpdateDateTime: new Date().toISOString(),
     pdrName: pdr.name,
-    processingEndDateTime: new Date().toString(),
-    processingStartDateTime: new Date().toString(),
-    productionDateTime: new Date().toString(),
+    processingEndDateTime: new Date().toISOString(),
+    processingStartDateTime: new Date().toISOString(),
+    productionDateTime: new Date().toISOString(),
     productVolume: '1000',
     provider: provider.name,
     published: true,
@@ -842,20 +842,20 @@ test.serial('writeGranulesFromMessage() on re-write saves granule records to Pos
   } = t.context;
 
   const completeGranule = fakeGranuleFactoryV2({
-    beginningDateTime: new Date().toString(),
+    beginningDateTime: new Date().toISOString(),
     cmrLink: 'example.com',
     collectionId: constructCollectionId(collection.name, collection.version),
     createdAt: Date.now(),
     duration: 1000,
-    endingDateTime: new Date().toString(),
+    endingDateTime: new Date().toISOString(),
     error: { errorKey: 'errorValue' },
     execution: executionUrl,
     files: files,
-    lastUpdateDateTime: new Date().toString(),
+    lastUpdateDateTime: new Date().toISOString(),
     pdrName: pdr.name,
-    processingEndDateTime: new Date().toString(),
-    processingStartDateTime: new Date().toString(),
-    productionDateTime: new Date().toString(),
+    processingEndDateTime: new Date().toISOString(),
+    processingStartDateTime: new Date().toISOString(),
+    productionDateTime: new Date().toISOString(),
     productVolume: '1000',
     provider: provider.name,
     published: true,
@@ -989,18 +989,18 @@ test.serial('writeGranulesFromMessage() on re-write saves granule records to Pos
   ];
 
   const completeGranule = fakeGranuleFactoryV2({
-    beginningDateTime: new Date().toString(),
+    beginningDateTime: new Date().toISOString(),
     cmrLink: 'example.com',
     collectionId: constructCollectionId(collection.name, collection.version),
     duration: 1000,
-    endingDateTime: new Date().toString(),
+    endingDateTime: new Date().toISOString(),
     error: { errorKey: 'errorValue' },
     files: files,
-    lastUpdateDateTime: new Date().toString(),
+    lastUpdateDateTime: new Date().toISOString(),
     pdrName: pdr.name,
-    processingEndDateTime: new Date().toString(),
-    processingStartDateTime: new Date().toString(),
-    productionDateTime: new Date().toString(),
+    processingEndDateTime: new Date().toISOString(),
+    processingStartDateTime: new Date().toISOString(),
+    productionDateTime: new Date().toISOString(),
     productVolume: '1000',
     provider: provider.name,
     published: true,
@@ -3968,21 +3968,21 @@ test.serial('writeGranuleFromApi() given a granule with all fields populated is
   ];
 
   const completeGranule = fakeGranuleFactoryV2({
-    beginningDateTime: new Date().toString(),
+    beginningDateTime: new Date().toISOString(),
     cmrLink: 'example.com',
     collectionId: constructCollectionId(collection.name, collection.version),
     createdAt: Date.now(),
     duration: 1000,
-    endingDateTime: new Date().toString(),
+    endingDateTime: new Date().toISOString(),
     error: { errorKey: 'errorValue' },
     execution: executionUrl,
     files: files,
     granuleId: granuleId,
-    lastUpdateDateTime: new Date().toString(),
+    lastUpdateDateTime: new Date().toISOString(),
     pdrName: pdr.name,
-    processingEndDateTime: new Date().toString(),
-    processingStartDateTime: new Date().toString(),
-    productionDateTime: new Date().toString(),
+    processingEndDateTime: new Date().toISOString(),
+    processingStartDateTime: new Date().toISOString(),
+    productionDateTime: new Date().toISOString(),
     productVolume: '1000',
     provider: provider.name,
     published: true,

diff --git a/packages/aws-client/tests/test-S3.js b/packages/aws-client/tests/test-S3.js
@@ -34,6 +34,7 @@ const {
   deleteS3Objects,
   promiseS3Upload,
   fileExists,
+  s3ObjectExists,
 } = require('../S3');
 const awsServices = require('../services');
 const { streamToString } = require('../test-utils');
@@ -501,3 +502,16 @@ test('fileExists() correctly returns false for non-existent file', async (t) =>
   const { Bucket } = t.context;
   t.false(await fileExists(Bucket, randomString()));
 });
+
+test('s3ObjectExists() returns true for existing file', async (t) => {
+  const { Bucket } = t.context;
+
+  const { Key } = await stageTestObjectToLocalStack(Bucket, 'asdf');
+  t.true(await s3ObjectExists({ Bucket, Key }));
+});
+
+test('s3ObjectExists() returns false for non-existent file', async (t) => {
+  const { Bucket } = t.context;
+  t.false(await s3ObjectExists({ Bucket, Key: randomString() }));
+  t.false(await s3ObjectExists({ Bucket: randomString(), Key: randomString() }));
+});
diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js
@@ -15,6 +15,7 @@ const {
   parseS3Uri,
   promiseS3Upload,
   s3GetObjectTagging,
+  s3ObjectExists,
   s3TagSetToQueryString,
   waitForObject,
   getObjectStreamContents,
@@ -1098,6 +1099,11 @@ async function getGranuleTemporalInfo(granule) {
 
   const cmrFilename = getS3UrlOfFile(cmrFile[0]);
 
+  if (!(await s3ObjectExists(parseS3Uri(cmrFilename)))) {
+    log.warn(`getGranuleTemporalInfo cmr file does not exist ${cmrFilename}`);
+    return {};
+  }
+
   if (isCMRISOFilename(cmrFilename)) {
     const metadata = await metadataObjectFromCMRXMLFile(cmrFilename);
     const metadataMI = get(

diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js
@@ -9,9 +9,11 @@ const { promisify } = require('util');
 const pickAll = require('lodash/fp/pickAll');
 const {
   buildS3Uri,
+  createBucket,
   getS3Object,
   parseS3Uri,
   promiseS3Upload,
+  putJsonS3Object,
   recursivelyDeleteS3Bucket,
   s3GetObjectTagging,
   s3TagSetToQueryString,
@@ -102,6 +104,12 @@ test.before(async (t) => {
   t.context.distributionBucketMap = Object.fromEntries(
     Object.values(bucketsJson).map(({ name }) => [name, name])
   );
+
+  t.context.cmrFileBucket = randomId('bucket');
+  await createBucket(t.context.cmrFileBucket);
+  await putJsonS3Object(t.context.cmrFileBucket, 'test.cmr.json', { foo: 'bar' });
+  await putJsonS3Object(t.context.cmrFileBucket, 'test.cmr.xml', { foo: 'bar' });
+  await putJsonS3Object(t.context.cmrFileBucket, 'test.cmr_iso.xml', { foo: 'bar' });
 });
 
 test.after.always(async (t) => {
@@ -114,6 +122,7 @@ test.after.always(async (t) => {
       SecretId: cmrPasswordSecret,
       ForceDeleteWithoutRecovery: true,
     }).promise(),
+    await recursivelyDeleteS3Bucket(t.context.cmrFileBucket),
   ]);
 
   t.context.launchpadStub.restore();
@@ -741,7 +750,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr.json',
       }],
     });
@@ -768,7 +777,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr.json',
       }],
     });
@@ -795,7 +804,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr.json',
       }],
     });
@@ -822,7 +831,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr.json',
       }],
     });
@@ -849,7 +858,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr.xml',
       }],
     });
@@ -876,7 +885,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr.xml',
       }],
     });
@@ -903,7 +912,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr_iso.xml',
       }],
     });
@@ -930,7 +939,7 @@ test.serial('getGranuleTemporalInfo returns temporal information from granule CM
     const temporalInfo = await getGranuleTemporalInfo({
       granuleId: 'testGranuleId',
       files: [{
-        bucket: 'bucket',
+        bucket: t.context.cmrFileBucket,
         key: 'test.cmr_iso.xml',
       }],
     });
@@ -953,6 +962,18 @@ test.serial('getGranuleTemporalInfo returns empty object if cmr file s3 url is n
   t.deepEqual(temporalInfo, {});
 });
 
+test.serial('getGranuleTemporalInfo returns empty object if cmr file s3 does not exist', async (t) => {
+  const temporalInfo = await getGranuleTemporalInfo({
+    granuleId: 'testGranuleId',
+    files: [{
+      path: t.context.cmrFileBucket,
+      name: 'not-exist.cmr_iso.xml',
+    }],
+  });
+
+  t.deepEqual(temporalInfo, {});
+});
+
 test.serial('generateFileUrl generates correct url for cmrGranuleUrlType distribution', (t) => {
   const filename = 's3://fake-bucket/folder/key.txt';
   const distEndpoint = 'www.example.com/';

diff --git a/packages/es-client/config/mappings.json b/packages/es-client/config/mappings.json
@@ -266,6 +266,10 @@
       "published": {
         "type": "boolean"
       },
+      "queryFields": {
+        "type": "object",
+        "dynamic": false
+      },
       "duration": {
         "type": "float"
       },
@@ -349,6 +353,10 @@
       "published": {
         "type": "boolean"
       },
+      "queryFields": {
+        "type": "object",
+        "dynamic": false
+      },
       "duration": {
         "type": "float"
       },

diff --git a/packages/message/src/Granules.ts b/packages/message/src/Granules.ts
@@ -8,10 +8,12 @@
  * const Granules = require('@cumulus/message/Granules');
  */
 
+import isEmpty from 'lodash/isEmpty';
 import isInteger from 'lodash/isInteger';
 import isUndefined from 'lodash/isUndefined';
 import mapValues from 'lodash/mapValues';
 import omitBy from 'lodash/omitBy';
+import pick from 'lodash/pick';
 
 import { CumulusMessageError } from '@cumulus/errors';
 import { Message } from '@cumulus/types';
@@ -171,7 +173,8 @@ function isGranuleTemporalInfo(
 }
 
 /**
- * Get granule temporal information from argument or directly from CMR.
+ * Get granule temporal information from argument, directly from CMR
+ * file or from granule object.
  *
  * Converts temporal information timestamps to a standardized ISO string
  * format for compatibility across database systems.
@@ -196,6 +199,11 @@ export const getGranuleCmrTemporalInfo = async ({
   const temporalInfo = isGranuleTemporalInfo(cmrTemporalInfo)
     ? { ...cmrTemporalInfo }
     : await cmrUtils.getGranuleTemporalInfo(granule);
+
+  if (isEmpty(temporalInfo)) {
+    return pick(granule, ['beginningDateTime', 'endingDateTime', 'productionDateTime', 'lastUpdateDateTime']);
+  }
+
   return mapValues(
     temporalInfo,
     convertDateToISOStringPreservingNull