CSD-91: Make the updateGranuleIdentifiers flag configurable in the update-granules-cmr-metadata-file-links task

CHANGELOG.md

@@ -96,6 +96,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
   - Resolved several integration issues with the granule-invalidator lambda.
   - Updated packaging script for granule-invalidator to use `uv pip install` instead of `uv sync`.
   - Added `private_api_lambda_arn` output to the archive module and `private_api_lambda_arn` variable to the ingest module.
+- **CSD-91**
+  - Added a task config var to update-granules-cmr-metadata-file-links `updateGranuleIdentifiers` for whether or not to update the Granule metadata's identifiers and `GranuleUR`, defaults to true. See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details.
 - **CSD-85**
   - Changed `update-granules-cmr-metadata-file-links` task config to accept a variable `excludeDataGranule`
     for whether or not to add or update a `Granule.DataGranule` to the granule's metadata, for users who do not want one added or updated from what their granule metadata already is (defaults to `false`). See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details.

packages/cmrjs/tests/cmr-utils/test-cmr-utils.js

@@ -491,8 +491,7 @@ test.serial('UpdateEcho10XMLMetadata updates GranuleUR and ProducerGranuleID cor
   );
   const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions);
   // Remove producerGranuleId from cmrMetadata.  We expect granuleUR = producerGranuleId
-  // in this case
-  // that granuleUR will after = granuleId, and producerGranuleId will be updated
+  // in this case that granuleUR will after = granuleId, and producerGranuleId will be updated
   const distEndpoint = 'https://distendpoint.com';
   const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' }));
 
@@ -517,6 +516,39 @@ test.serial('UpdateEcho10XMLMetadata updates GranuleUR and ProducerGranuleID cor
   t.is(metadataObject.Granule.DataGranule.get('ProducerGranuleId'), 'TestFixtureGranuleUR');
 });
 
+test.serial('UpdateEcho10XMLMetadata should not update GranuleUR and ProducerGranuleID when updateGranuleIdentifiers is false', async (t) => {
+  const { bucketTypes, distributionBucketMap } = t.context;
+  const cmrXml = await fs.readFile(
+    path.join(__dirname, '../fixtures/cmrFileUpdateFixture.cmr.xml'),
+    'utf8'
+  );
+  const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions);
+  // Remove producerGranuleId from cmrMetadata.  We expect granuleUR = producerGranuleId
+  // in this case that granuleUR will after = granuleId, and producerGranuleId will be updated
+  const distEndpoint = 'https://distendpoint.com';
+  const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' }));
+
+  const { metadataObject } = await updateEcho10XMLMetadata({
+    cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' },
+    files: [],
+    distEndpoint,
+    bucketTypes,
+    distributionBucketMap,
+    testOverrides: {
+      generateEcho10XMLStringMethod: () => 'testXmlString',
+      metadataObjectFromCMRXMLFileMethod: () => cmrMetadata,
+      uploadEcho10CMRFileMethod: uploadEchoSpy,
+    },
+    updateGranuleIdentifiers: false,
+    granuleId: 'TestFixtureGranuleUR_uniq',
+    producerGranuleId: 'TestFixtureGranuleUR',
+    excludeDataGranule: false,
+  });
+
+  t.deepEqual(metadataObject.Granule.GranuleUR, cmrMetadata.Granule.GranuleUR);
+  t.deepEqual(metadataObject.Granule.DataGranule, cmrMetadata.Granule.DataGranule);
+});
+
 test.serial('UpdateEcho10XMLMetadata does not update granule DataGranule metadata when excludeDataGranule is true', async (t) => {
   const { bucketTypes, distributionBucketMap } = t.context;
   const cmrXml = await fs.readFile(
@@ -595,7 +627,7 @@ test.serial('UpdateEcho10XMLMetadata maintains ECHO10 DataGranule element order'
   t.is(metadataObject.Granule.DataGranule.get('ProducerGranuleId'), 'NEW_PRODUCER_ID');
 });
 
-test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is true', async (t) => {
+test.serial('updateUMMGMetadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is true', async (t) => {
   const { bucketTypes, distributionBucketMap } = t.context;
 
   const distEndpoint = 'https://distendpoint.com';
@@ -629,7 +661,42 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct
   t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR');
 });
 
-test.serial('updateUMMG Metadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => {
+test.serial('updateUMMGMetadata does not update Granule metadata when updateGranuleIdentifiers is false', async (t) => {
+  const { bucketTypes, distributionBucketMap } = t.context;
+
+  const distEndpoint = 'https://distendpoint.com';
+  const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' }));
+
+  const cmrJSON = await fs.readFile(
+    path.join(__dirname, '../fixtures/MOD09GQ.A3411593.1itJ_e.006.9747594822314_v1.6.2.cmr.json'),
+    'utf8'
+  );
+  const cmrMetadata = JSON.parse(cmrJSON);
+  const filesObject = await readJsonFixture(
+    path.join(__dirname, '../fixtures/UMMGFilesObjectFixture.json')
+  );
+
+  const { metadataObject } = await updateUMMGMetadata({
+    granuleId: 'TestFixtureGranuleUR_uniq',
+    producerGranuleId: 'TestFixtureGranuleUR',
+    excludeDataGranule: false,
+    cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' },
+    files: filesObject,
+    distEndpoint,
+    bucketTypes,
+    distributionBucketMap,
+    updateGranuleIdentifiers: false,
+    testOverrides: {
+      uploadUMMGJSONCMRFileMethod: uploadEchoSpy,
+      metadataObjectFromCMRJSONFileMethod: () => cmrMetadata,
+    },
+  });
+
+  t.deepEqual(metadataObject.GranuleUR, cmrMetadata.GranuleUR);
+  t.deepEqual(metadataObject.DataGranule, cmrMetadata.DataGranule);
+});
+
+test.serial('updateUMMGMetadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => {
   const { bucketTypes, distributionBucketMap } = t.context;
 
   const distEndpoint = 'https://distendpoint.com';

tasks/update-granules-cmr-metadata-file-links/README.md

@@ -11,6 +11,8 @@ This Cumulus task component updates CMR metadata files to have correct values fo
 
 **Note** As default behavior for this task, for UMMG and ECHO10 granules, a `Granule.DataGranule` section will be added if not already present within the metadata. For UMMG granules (as of `CSD-85`), the required fields ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/unified-metadata-model/browse/granule/v1.6.6/umm-g-json-schema.json#257)) of `ProductionDateTime` and `DayNightFlag` will be populated with default values (the time the task is ran for `ProductionDateTime` and `Unspecified` for `DayNightFlag`) if they are not already included, along with the other updates described above. For ECHO10 granules, the required field for DataGranules, `ProducerGranuleId` ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/echo-schemas/browse/schemas/10.0/Granule.xsd#409)), will be populated, along with the other updates described above. To disable adding/updating the granule's metadata in this task for both ECHO10 and UMMG granules, set `excludeDataGranule` as `true` (boolean, not a string) (added as part of `CSD-85`) in the task config schema. If set to `true` this task will not change anything relating to the `Granule.DataGranule` in the granule's metadata, no adding or updating.
 
+**Note** To disable updating the granule metadata's identifiers and granuleUr in this task for both ECHO10 and UMMG, set `updateGranuleIdentifiers` to `false` (boolean, not a string) in the task config schema (added as part of `CSD-91`). By default, this task sets `updateGranuleIdentifiers` to `true` for both ECHO10 and UMMG.
+
 ## Input/Output Schema
 
 ### Input

tasks/update-granules-cmr-metadata-file-links/index.js

@@ -38,7 +38,9 @@ const logger = new Logger({ sender: '@cumulus/update-granules-cmr-metadata-file-
  *                                           (e.g. { bucket: distribution path })
  * @param {Object} excludeFileRegexPattern - pattern by which to exclude files from processing
  * @param {boolean} excludeDataGranule       - Whether to add or update the DataGranule
- * node in the granule's metadata
+ *                                             node in the granule's metadata
+ * @param {boolean} updateGranuleIdentifiers - Whether to update the Granule's Identifiers
+ *                                             and granuleUR
  * @returns {Promise<Object[]>} Array of updated CMR files with etags of newly updated files.
  *
  */
@@ -51,7 +53,8 @@ async function updateEachCmrFileMetadata(
   bucketTypes,
   distributionBucketMap,
   excludeFileRegexPattern,
-  excludeDataGranule
+  excludeDataGranule,
+  updateGranuleIdentifiers
 ) {
   return await Promise.all(cmrFiles.map(async (cmrFile) => {
     const granuleId = cmrFile.granuleId;
@@ -75,7 +78,7 @@ async function updateEachCmrFileMetadata(
       bucketTypes,
       cmrGranuleUrlType,
       distributionBucketMap,
-      updateGranuleIdentifiers: true,
+      updateGranuleIdentifiers,
       excludeDataGranule,
     });
   }));
@@ -118,6 +121,8 @@ async function updateGranulesCmrMetadata(event) {
   const granulesByGranuleId = keyBy(granules, 'granuleId');
   const excludeDataGranule = isBoolean(event.config.excludeDataGranule) ?
     event.config.excludeDataGranule : false;
+  const updateGranuleIdentifiers = isBoolean(event.config.updateGranuleIdentifiers) ?
+    event.config.updateGranuleIdentifiers : true;
 
   const distributionBucketMap = await fetchDistributionBucketMap();
   const updatedCmrFiles = await updateEachCmrFileMetadata(
@@ -128,7 +133,8 @@ async function updateGranulesCmrMetadata(event) {
     bucketTypes,
     distributionBucketMap,
     config.excludeFileRegex,
-    excludeDataGranule
+    excludeDataGranule,
+    updateGranuleIdentifiers
   );
 
   const updatedGranulesByGranuleId = await updateCmrFileInfo(cmrFiles, granulesByGranuleId);

tasks/update-granules-cmr-metadata-file-links/schemas/config.json

@@ -47,8 +47,13 @@
     },
     "excludeDataGranule": {
       "type": "boolean",
-      "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. if True, will disable updates",
+      "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. If true, will disable updates",
       "default": false
+    },
+    "updateGranuleIdentifiers": {
+      "type": "boolean",
+      "description": "Flag to set exclusion behavior of Identifier and granuleUr metadata updates as part of this task. If false, will disable updates",
+      "default": true
     }
   }
 }

tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Granule>
-    <GranuleUR>MOD11A1.A2017200.h19v04.006.2017201090724</GranuleUR>
+    <GranuleUR>TestFixtureGranuleUR_uniq</GranuleUR>
     <InsertTime>2017-11-20T23:02:40.055807</InsertTime>
     <LastUpdate>2017-11-20T23:02:40.055814</LastUpdate>
     <Collection>

tasks/update-granules-cmr-metadata-file-links/tests/test-index.js

@@ -4,6 +4,8 @@ const fs = require('fs');
 const path = require('path');
 const test = require('ava');
 const cloneDeep = require('lodash/cloneDeep');
+const xml2js = require('xml2js');
+const { promisify } = require('util');
 
 const {
   buildS3Uri,
@@ -21,10 +23,25 @@ const {
 const { s3 } = require('@cumulus/aws-client/services');
 const { getDistributionBucketMapKey } = require('@cumulus/distribution-utils');
 
-const { isCMRFile } = require('@cumulus/cmrjs');
+const { isCMRFile, xmlParseOptions } = require('@cumulus/cmrjs');
 
 const { updateGranulesCmrMetadata, updateCmrFileInfo } = require('..');
 
+function getGranuleURValue(obj) {
+  const value = obj.GranuleUR ?? obj.Granule?.GranuleUR ?? '';
+  // Handle xml2js array wrapping (it converts values to arrays by default)
+  return Array.isArray(value) ? value[0] : value;
+}
+
+async function getCMRFileBodyContent(cmrFile) {
+  const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key });
+  const payloadContents = await getObjectStreamContents(payloadResponse.Body);
+  if (cmrFile.key.endsWith('.xml')) {
+    return await promisify(xml2js.parseString)(payloadContents, xmlParseOptions);
+  }
+  return JSON.parse(payloadContents);
+}
+
 function cmrReadStream(file) {
   return file.endsWith('.cmr.xml') ? fs.createReadStream('tests/data/meta.xml') : fs.createReadStream('tests/data/ummg-meta.json');
 }
@@ -335,3 +352,113 @@ test('updateCmrFileInfo - throws error when CMR file not found', async (t) => {
     message: 'CMR file not found for granule with ID granule1',
   });
 });
+
+test.serial('GranuleUr and Identifiers do not get updated when config variable to updateGranuleIdentifier flag is false', async (t) => {
+  const newPayload = buildPayload(t);
+
+  newPayload.input.granules.forEach((granule) => {
+    const newFile = {
+      bucket: t.context.publicBucket,
+      key: 'some/prefix/some_filename.json',
+      type: 'data',
+    };
+    granule.files.push(newFile);
+  });
+  newPayload.config.updateGranuleIdentifiers = false;
+
+  await validateConfig(t, newPayload.config);
+  await validateInput(t, newPayload.input);
+
+  const filesToUpload = cloneDeep(t.context.filesToUpload);
+  await uploadFiles(filesToUpload, t.context.stagingBucket);
+  const preUpdateCmrFiles = [];
+  newPayload.input.granules.forEach((granule) => {
+    granule.files.forEach((file) => {
+      if (isCMRFile(file)) {
+        file.granuleId = granule.granuleId;
+        preUpdateCmrFiles.push(file);
+      }
+    });
+  });
+
+  const preUpdateBodyContent = [];
+  await Promise.all(preUpdateCmrFiles.map(async (cmrFile) => {
+    preUpdateBodyContent.push(await getCMRFileBodyContent(cmrFile));
+  }));
+
+  await updateGranulesCmrMetadata(newPayload);
+  const cmrFiles = [];
+  newPayload.input.granules.forEach((granule) => {
+    granule.files.forEach((file) => {
+      if (isCMRFile(file)) {
+        file.granuleId = granule.granuleId;
+        cmrFiles.push(file);
+      }
+    });
+  });
+
+  const updatedBodyContent = [];
+  await Promise.all(cmrFiles.map(async (cmrFile) => {
+    updatedBodyContent.push(await getCMRFileBodyContent(cmrFile));
+  }));
+
+  // sort to avoid test assertion order issues
+  preUpdateBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b)));
+  updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b)));
+  t.is(updatedBodyContent[0].GranuleUR, preUpdateBodyContent[0].GranuleUR);
+  t.is(updatedBodyContent[1].GranuleUR, preUpdateBodyContent[1].GranuleUR);
+  t.deepEqual(updatedBodyContent[2].Granule.GranuleUR,
+    updatedBodyContent[2].Granule.GranuleUR);
+  t.deepEqual(updatedBodyContent[0].DataGranule, preUpdateBodyContent[0].DataGranule);
+  t.deepEqual(updatedBodyContent[1].DataGranule, preUpdateBodyContent[1].DataGranule);
+  t.deepEqual(updatedBodyContent[2].Granule.DataGranule,
+    updatedBodyContent[2].Granule.DataGranule);
+});
+
+test.serial('GranuleUr and Identifiers get updated when config variable to updateGranuleIdentifier flag is true', async (t) => {
+  const newPayload = buildPayload(t);
+  newPayload.input.granules.sort((a, b) => a.granuleId.localeCompare(b.granuleId));
+  newPayload.input.granules.forEach((granule) => {
+    const newFile = {
+      bucket: t.context.publicBucket,
+      key: 'some/prefix/some_filename.json',
+      type: 'data',
+    };
+    granule.files.push(newFile);
+  });
+  newPayload.config.updateGranuleIdentifiers = true;
+
+  await validateConfig(t, newPayload.config);
+  await validateInput(t, newPayload.input);
+
+  const filesToUpload = cloneDeep(t.context.filesToUpload);
+  await uploadFiles(filesToUpload, t.context.stagingBucket);
+
+  await updateGranulesCmrMetadata(newPayload);
+
+  const cmrFiles = [];
+  newPayload.input.granules.forEach((granule) => {
+    granule.files.forEach((file) => {
+      if (isCMRFile(file)) {
+        cmrFiles.push(file);
+      }
+    });
+  });
+
+  const updatedBodyContent = [];
+  await Promise.all(cmrFiles.map(async (cmrFile) => {
+    updatedBodyContent.push(await getCMRFileBodyContent(cmrFile));
+  }));
+
+  updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b)));
+  t.is(updatedBodyContent[0].GranuleUR, newPayload.input.granules[0].granuleId);
+  t.is(updatedBodyContent[1].GranuleUR, newPayload.input.granules[1].granuleId);
+  t.is(updatedBodyContent[2].Granule.GranuleUR[0],
+    newPayload.input.granules[2].granuleId);
+  t.is(updatedBodyContent[0].DataGranule.Identifiers[0].Identifier,
+    newPayload.input.granules[0].granuleId);
+  t.is(updatedBodyContent[1].DataGranule.Identifiers[0].Identifier,
+    newPayload.input.granules[1].granuleId);
+  t.is(updatedBodyContent[2].Granule.DataGranule[0].ProducerGranuleId[0],
+    newPayload.input.granules[2].granuleId);
+});