diff --git a/CHANGELOG.md b/CHANGELOG.md index 89171ab83d1f..802186c0e9a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Resolved several integration issues with the granule-invalidator lambda. - Updated packaging script for granule-invalidator to use `uv pip install` instead of `uv sync`. - Added `private_api_lambda_arn` output to the archive module and `private_api_lambda_arn` variable to the ingest module. +- **CSD-91** + - Added a task config var to update-granules-cmr-metadata-file-links `updateGranuleIdentifiers` for whether or not to update the Granule metadata's identifiers and `GranuleUR`, defaults to true. See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details. - **CSD-85** - Changed `update-granules-cmr-metadata-file-links` task config to accept a variable `excludeDataGranule` for whether or not to add or update a `Granule.DataGranule` to the granule's metadata, for users who do not want one added or updated from what their granule metadata already is (defaults to `false`). See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details. diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index 4d741e12fef0..4df4467a4406 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -491,8 +491,7 @@ test.serial('UpdateEcho10XMLMetadata updates GranuleUR and ProducerGranuleID cor ); const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions); // Remove producerGranuleId from cmrMetadata. We expect granuleUR = producerGranuleId - // in this case - // that granuleUR will after = granuleId, and producerGranuleId will be updated + // in this case that granuleUR will after = granuleId, and producerGranuleId will be updated const distEndpoint = 'https://distendpoint.com'; const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); @@ -517,6 +516,39 @@ test.serial('UpdateEcho10XMLMetadata updates GranuleUR and ProducerGranuleID cor t.is(metadataObject.Granule.DataGranule.get('ProducerGranuleId'), 'TestFixtureGranuleUR'); }); +test.serial('UpdateEcho10XMLMetadata should not update GranuleUR and ProducerGranuleID when updateGranuleIdentifiers is false', async (t) => { + const { bucketTypes, distributionBucketMap } = t.context; + const cmrXml = await fs.readFile( + path.join(__dirname, '../fixtures/cmrFileUpdateFixture.cmr.xml'), + 'utf8' + ); + const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions); + // Remove producerGranuleId from cmrMetadata. We expect granuleUR = producerGranuleId + // in this case that granuleUR will after = granuleId, and producerGranuleId will be updated + const distEndpoint = 'https://distendpoint.com'; + const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); + + const { metadataObject } = await updateEcho10XMLMetadata({ + cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, + files: [], + distEndpoint, + bucketTypes, + distributionBucketMap, + testOverrides: { + generateEcho10XMLStringMethod: () => 'testXmlString', + metadataObjectFromCMRXMLFileMethod: () => cmrMetadata, + uploadEcho10CMRFileMethod: uploadEchoSpy, + }, + updateGranuleIdentifiers: false, + granuleId: 'TestFixtureGranuleUR_uniq', + producerGranuleId: 'TestFixtureGranuleUR', + excludeDataGranule: false, + }); + + t.deepEqual(metadataObject.Granule.GranuleUR, cmrMetadata.Granule.GranuleUR); + t.deepEqual(metadataObject.Granule.DataGranule, cmrMetadata.Granule.DataGranule); +}); + test.serial('UpdateEcho10XMLMetadata does not update granule DataGranule metadata when excludeDataGranule is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const cmrXml = await fs.readFile( @@ -595,7 +627,7 @@ test.serial('UpdateEcho10XMLMetadata maintains ECHO10 DataGranule element order' t.is(metadataObject.Granule.DataGranule.get('ProducerGranuleId'), 'NEW_PRODUCER_ID'); }); -test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is true', async (t) => { +test.serial('updateUMMGMetadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const distEndpoint = 'https://distendpoint.com'; @@ -629,7 +661,42 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); }); -test.serial('updateUMMG Metadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => { +test.serial('updateUMMGMetadata does not update Granule metadata when updateGranuleIdentifiers is false', async (t) => { + const { bucketTypes, distributionBucketMap } = t.context; + + const distEndpoint = 'https://distendpoint.com'; + const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); + + const cmrJSON = await fs.readFile( + path.join(__dirname, '../fixtures/MOD09GQ.A3411593.1itJ_e.006.9747594822314_v1.6.2.cmr.json'), + 'utf8' + ); + const cmrMetadata = JSON.parse(cmrJSON); + const filesObject = await readJsonFixture( + path.join(__dirname, '../fixtures/UMMGFilesObjectFixture.json') + ); + + const { metadataObject } = await updateUMMGMetadata({ + granuleId: 'TestFixtureGranuleUR_uniq', + producerGranuleId: 'TestFixtureGranuleUR', + excludeDataGranule: false, + cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, + files: filesObject, + distEndpoint, + bucketTypes, + distributionBucketMap, + updateGranuleIdentifiers: false, + testOverrides: { + uploadUMMGJSONCMRFileMethod: uploadEchoSpy, + metadataObjectFromCMRJSONFileMethod: () => cmrMetadata, + }, + }); + + t.deepEqual(metadataObject.GranuleUR, cmrMetadata.GranuleUR); + t.deepEqual(metadataObject.DataGranule, cmrMetadata.DataGranule); +}); + +test.serial('updateUMMGMetadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const distEndpoint = 'https://distendpoint.com'; diff --git a/tasks/update-granules-cmr-metadata-file-links/README.md b/tasks/update-granules-cmr-metadata-file-links/README.md index 36bd798965ab..91eac7cd57f9 100644 --- a/tasks/update-granules-cmr-metadata-file-links/README.md +++ b/tasks/update-granules-cmr-metadata-file-links/README.md @@ -11,6 +11,8 @@ This Cumulus task component updates CMR metadata files to have correct values fo **Note** As default behavior for this task, for UMMG and ECHO10 granules, a `Granule.DataGranule` section will be added if not already present within the metadata. For UMMG granules (as of `CSD-85`), the required fields ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/unified-metadata-model/browse/granule/v1.6.6/umm-g-json-schema.json#257)) of `ProductionDateTime` and `DayNightFlag` will be populated with default values (the time the task is ran for `ProductionDateTime` and `Unspecified` for `DayNightFlag`) if they are not already included, along with the other updates described above. For ECHO10 granules, the required field for DataGranules, `ProducerGranuleId` ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/echo-schemas/browse/schemas/10.0/Granule.xsd#409)), will be populated, along with the other updates described above. To disable adding/updating the granule's metadata in this task for both ECHO10 and UMMG granules, set `excludeDataGranule` as `true` (boolean, not a string) (added as part of `CSD-85`) in the task config schema. If set to `true` this task will not change anything relating to the `Granule.DataGranule` in the granule's metadata, no adding or updating. +**Note** To disable updating the granule metadata's identifiers and granuleUr in this task for both ECHO10 and UMMG, set `updateGranuleIdentifiers` to `false` (boolean, not a string) in the task config schema (added as part of `CSD-91`). By default, this task sets `updateGranuleIdentifiers` to `true` for both ECHO10 and UMMG. + ## Input/Output Schema ### Input diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index 203e6e0ea1a8..6f75f538ad93 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -38,7 +38,9 @@ const logger = new Logger({ sender: '@cumulus/update-granules-cmr-metadata-file- * (e.g. { bucket: distribution path }) * @param {Object} excludeFileRegexPattern - pattern by which to exclude files from processing * @param {boolean} excludeDataGranule - Whether to add or update the DataGranule - * node in the granule's metadata + * node in the granule's metadata + * @param {boolean} updateGranuleIdentifiers - Whether to update the Granule's Identifiers + * and granuleUR * @returns {Promise} Array of updated CMR files with etags of newly updated files. * */ @@ -51,7 +53,8 @@ async function updateEachCmrFileMetadata( bucketTypes, distributionBucketMap, excludeFileRegexPattern, - excludeDataGranule + excludeDataGranule, + updateGranuleIdentifiers ) { return await Promise.all(cmrFiles.map(async (cmrFile) => { const granuleId = cmrFile.granuleId; @@ -75,7 +78,7 @@ async function updateEachCmrFileMetadata( bucketTypes, cmrGranuleUrlType, distributionBucketMap, - updateGranuleIdentifiers: true, + updateGranuleIdentifiers, excludeDataGranule, }); })); @@ -118,6 +121,8 @@ async function updateGranulesCmrMetadata(event) { const granulesByGranuleId = keyBy(granules, 'granuleId'); const excludeDataGranule = isBoolean(event.config.excludeDataGranule) ? event.config.excludeDataGranule : false; + const updateGranuleIdentifiers = isBoolean(event.config.updateGranuleIdentifiers) ? + event.config.updateGranuleIdentifiers : true; const distributionBucketMap = await fetchDistributionBucketMap(); const updatedCmrFiles = await updateEachCmrFileMetadata( @@ -128,7 +133,8 @@ async function updateGranulesCmrMetadata(event) { bucketTypes, distributionBucketMap, config.excludeFileRegex, - excludeDataGranule + excludeDataGranule, + updateGranuleIdentifiers ); const updatedGranulesByGranuleId = await updateCmrFileInfo(cmrFiles, granulesByGranuleId); diff --git a/tasks/update-granules-cmr-metadata-file-links/schemas/config.json b/tasks/update-granules-cmr-metadata-file-links/schemas/config.json index 5340abb836c9..a5648a834684 100644 --- a/tasks/update-granules-cmr-metadata-file-links/schemas/config.json +++ b/tasks/update-granules-cmr-metadata-file-links/schemas/config.json @@ -47,8 +47,13 @@ }, "excludeDataGranule": { "type": "boolean", - "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. if True, will disable updates", + "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. If true, will disable updates", "default": false + }, + "updateGranuleIdentifiers": { + "type": "boolean", + "description": "Flag to set exclusion behavior of Identifier and granuleUr metadata updates as part of this task. If false, will disable updates", + "default": true } } } diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml b/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml index 8c2a07bf41de..793481c8548c 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml +++ b/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml @@ -1,6 +1,6 @@ - MOD11A1.A2017200.h19v04.006.2017201090724 + TestFixtureGranuleUR_uniq 2017-11-20T23:02:40.055807 2017-11-20T23:02:40.055814 diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js index 21fc898271dc..3653fcd6085c 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js +++ b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js @@ -4,6 +4,8 @@ const fs = require('fs'); const path = require('path'); const test = require('ava'); const cloneDeep = require('lodash/cloneDeep'); +const xml2js = require('xml2js'); +const { promisify } = require('util'); const { buildS3Uri, @@ -21,10 +23,25 @@ const { const { s3 } = require('@cumulus/aws-client/services'); const { getDistributionBucketMapKey } = require('@cumulus/distribution-utils'); -const { isCMRFile } = require('@cumulus/cmrjs'); +const { isCMRFile, xmlParseOptions } = require('@cumulus/cmrjs'); const { updateGranulesCmrMetadata, updateCmrFileInfo } = require('..'); +function getGranuleURValue(obj) { + const value = obj.GranuleUR ?? obj.Granule?.GranuleUR ?? ''; + // Handle xml2js array wrapping (it converts values to arrays by default) + return Array.isArray(value) ? value[0] : value; +} + +async function getCMRFileBodyContent(cmrFile) { + const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); + const payloadContents = await getObjectStreamContents(payloadResponse.Body); + if (cmrFile.key.endsWith('.xml')) { + return await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); + } + return JSON.parse(payloadContents); +} + function cmrReadStream(file) { return file.endsWith('.cmr.xml') ? fs.createReadStream('tests/data/meta.xml') : fs.createReadStream('tests/data/ummg-meta.json'); } @@ -335,3 +352,113 @@ test('updateCmrFileInfo - throws error when CMR file not found', async (t) => { message: 'CMR file not found for granule with ID granule1', }); }); + +test.serial('GranuleUr and Identifiers do not get updated when config variable to updateGranuleIdentifier flag is false', async (t) => { + const newPayload = buildPayload(t); + + newPayload.input.granules.forEach((granule) => { + const newFile = { + bucket: t.context.publicBucket, + key: 'some/prefix/some_filename.json', + type: 'data', + }; + granule.files.push(newFile); + }); + newPayload.config.updateGranuleIdentifiers = false; + + await validateConfig(t, newPayload.config); + await validateInput(t, newPayload.input); + + const filesToUpload = cloneDeep(t.context.filesToUpload); + await uploadFiles(filesToUpload, t.context.stagingBucket); + const preUpdateCmrFiles = []; + newPayload.input.granules.forEach((granule) => { + granule.files.forEach((file) => { + if (isCMRFile(file)) { + file.granuleId = granule.granuleId; + preUpdateCmrFiles.push(file); + } + }); + }); + + const preUpdateBodyContent = []; + await Promise.all(preUpdateCmrFiles.map(async (cmrFile) => { + preUpdateBodyContent.push(await getCMRFileBodyContent(cmrFile)); + })); + + await updateGranulesCmrMetadata(newPayload); + const cmrFiles = []; + newPayload.input.granules.forEach((granule) => { + granule.files.forEach((file) => { + if (isCMRFile(file)) { + file.granuleId = granule.granuleId; + cmrFiles.push(file); + } + }); + }); + + const updatedBodyContent = []; + await Promise.all(cmrFiles.map(async (cmrFile) => { + updatedBodyContent.push(await getCMRFileBodyContent(cmrFile)); + })); + + // sort to avoid test assertion order issues + preUpdateBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b))); + updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b))); + t.is(updatedBodyContent[0].GranuleUR, preUpdateBodyContent[0].GranuleUR); + t.is(updatedBodyContent[1].GranuleUR, preUpdateBodyContent[1].GranuleUR); + t.deepEqual(updatedBodyContent[2].Granule.GranuleUR, + updatedBodyContent[2].Granule.GranuleUR); + t.deepEqual(updatedBodyContent[0].DataGranule, preUpdateBodyContent[0].DataGranule); + t.deepEqual(updatedBodyContent[1].DataGranule, preUpdateBodyContent[1].DataGranule); + t.deepEqual(updatedBodyContent[2].Granule.DataGranule, + updatedBodyContent[2].Granule.DataGranule); +}); + +test.serial('GranuleUr and Identifiers get updated when config variable to updateGranuleIdentifier flag is true', async (t) => { + const newPayload = buildPayload(t); + newPayload.input.granules.sort((a, b) => a.granuleId.localeCompare(b.granuleId)); + newPayload.input.granules.forEach((granule) => { + const newFile = { + bucket: t.context.publicBucket, + key: 'some/prefix/some_filename.json', + type: 'data', + }; + granule.files.push(newFile); + }); + newPayload.config.updateGranuleIdentifiers = true; + + await validateConfig(t, newPayload.config); + await validateInput(t, newPayload.input); + + const filesToUpload = cloneDeep(t.context.filesToUpload); + await uploadFiles(filesToUpload, t.context.stagingBucket); + + await updateGranulesCmrMetadata(newPayload); + + const cmrFiles = []; + newPayload.input.granules.forEach((granule) => { + granule.files.forEach((file) => { + if (isCMRFile(file)) { + cmrFiles.push(file); + } + }); + }); + + const updatedBodyContent = []; + await Promise.all(cmrFiles.map(async (cmrFile) => { + updatedBodyContent.push(await getCMRFileBodyContent(cmrFile)); + })); + + updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b))); + t.is(updatedBodyContent[0].GranuleUR, newPayload.input.granules[0].granuleId); + t.is(updatedBodyContent[1].GranuleUR, newPayload.input.granules[1].granuleId); + t.is(updatedBodyContent[2].Granule.GranuleUR[0], + newPayload.input.granules[2].granuleId); + t.is(updatedBodyContent[0].DataGranule.Identifiers[0].Identifier, + newPayload.input.granules[0].granuleId); + t.is(updatedBodyContent[1].DataGranule.Identifiers[0].Identifier, + newPayload.input.granules[1].granuleId); + t.is(updatedBodyContent[2].Granule.DataGranule[0].ProducerGranuleId[0], + newPayload.input.granules[2].granuleId); +});