From e3e4b1c6f8a037d8f5b87bba5e47f940f7bf4e72 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Tue, 3 Feb 2026 11:44:24 -0800 Subject: [PATCH 01/26] add new env variable update_granule_identifiers to pass to updateCMRMetadata --- tasks/update-granules-cmr-metadata-file-links/index.js | 2 +- .../ingest/update-granules-cmr-metadata-file-links-task.tf | 1 + tf-modules/ingest/variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index 3cd5cfef2281..d07eed11a40e 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -71,7 +71,7 @@ async function updateEachCmrFileMetadata( bucketTypes, cmrGranuleUrlType, distributionBucketMap, - updateGranuleIdentifiers: true, + updateGranuleIdentifiers: process.env.update_granule_identifiers, }); })); } diff --git a/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf b/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf index 24f9ec35538f..f77c8cea1b04 100644 --- a/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf +++ b/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf @@ -17,6 +17,7 @@ resource "aws_lambda_function" "update_granules_cmr_metadata_file_links_task" { stackName = var.prefix CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" system_bucket = var.system_bucket + update_granule_identifiers = var.default_update_granule_identifiers } } diff --git a/tf-modules/ingest/variables.tf b/tf-modules/ingest/variables.tf index 183b15c4896e..fc8e257efdf2 100644 --- a/tf-modules/ingest/variables.tf +++ b/tf-modules/ingest/variables.tf @@ -203,6 +203,12 @@ variable "default_log_retention_days" { description = "Optional default value that user chooses for their log retention periods" } +variable "default_update_granule_identifiers" { + description = "Whether or not to automatically update granule identifiers" + type = bool + default = true +} + variable "sync_granule_s3_jitter_max_ms" { description = "Maximum random jitter in milliseconds to apply before S3 operations in SyncGranule task (0-59000). Set to 0 to disable jitter." type = number From bd70a245e66ca4f3bfe3fb4ae3c5dd5f0d9693ba Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Wed, 11 Feb 2026 22:59:37 -0800 Subject: [PATCH 02/26] * add default_update_granule_identifiers env var to cumulus-tf and tf-modules * update update-granule-cmr-metadata-file-links updateEachCmrFileMetadata function to pass update_granule_identifiers env var --- example/cumulus-tf/main.tf | 3 + example/cumulus-tf/variables.tf | 6 ++ packages/cmrjs/src/cmr-utils.js | 4 +- .../cmrjs/tests/cmr-utils/test-cmr-utils.js | 58 +++++++++++++++++++ tf-modules/cumulus/ingest.tf | 3 + tf-modules/cumulus/variables.tf | 6 ++ 6 files changed, 78 insertions(+), 2 deletions(-) diff --git a/example/cumulus-tf/main.tf b/example/cumulus-tf/main.tf index a9670c19ff5b..19f5e348ee2b 100644 --- a/example/cumulus-tf/main.tf +++ b/example/cumulus-tf/main.tf @@ -124,6 +124,9 @@ module "cumulus" { default_s3_multipart_chunksize_mb = var.default_s3_multipart_chunksize_mb sync_granule_s3_jitter_max_ms = var.sync_granule_s3_jitter_max_ms + # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not + update_granule_identifiers = var.default_update_granule_identifiers + launchpad_api = var.launchpad_api launchpad_certificate = var.launchpad_certificate launchpad_passphrase = var.launchpad_passphrase diff --git a/example/cumulus-tf/variables.tf b/example/cumulus-tf/variables.tf index 680c5e8c95cf..03c92b259971 100644 --- a/example/cumulus-tf/variables.tf +++ b/example/cumulus-tf/variables.tf @@ -211,6 +211,12 @@ variable "tea_distribution_url" { default = null } +variable "default_update_granule_identifiers" { + description = "Whether or not to automatically update granule identifiers" + type = bool + default = true +} + variable "sync_granule_s3_jitter_max_ms" { description = "Maximum random jitter in milliseconds for SyncGranule S3 operations (0-59000). Set to 0 to disable." type = number diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js index 0502360d937e..a6bf1e519306 100644 --- a/packages/cmrjs/src/cmr-utils.js +++ b/packages/cmrjs/src/cmr-utils.js @@ -889,7 +889,7 @@ async function updateUMMGMetadata({ distributionBucketMap, producerGranuleId, granuleId, - updateGranuleIdentifiers = false, + updateGranuleIdentifiers, testOverrides = {}, }) { const { @@ -1223,7 +1223,7 @@ async function updateCMRMetadata({ published, bucketTypes, cmrGranuleUrlType = 'both', - updateGranuleIdentifiers = false, + updateGranuleIdentifiers, distributionBucketMap, testOverrides = {}, }) { diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index fead58fc74cc..c49615ba0418 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -593,6 +593,64 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); }); +test.only('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is using update_granule_identifiers environment variable', async (t) => { + const { bucketTypes, distributionBucketMap } = t.context; + const originalUpdateGranuleIdentifiers = process.env.update_granule_identifiers; + process.env.update_granule_identifiers = false; + + // Yes, ETag values always include enclosing double-quotes + const distEndpoint = 'https://distendpoint.com'; + const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); + + const cmrJSON = await fs.readFile( + path.join(__dirname, '../fixtures/MOD09GQ.A3411593.1itJ_e.006.9747594822314_v1.6.2.cmr.json'), + 'utf8' + ); + const cmrMetadata = JSON.parse(cmrJSON); + const filesObject = await readJsonFixture( + path.join(__dirname, '../fixtures/UMMGFilesObjectFixture.json') + ); + + const { metadataObject } = await updateUMMGMetadata({ + granuleId: 'TestFixtureGranuleUR_uniq', + producerGranuleId: 'TestFixtureGranuleUR', + cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, + files: filesObject, + distEndpoint, + bucketTypes, + distributionBucketMap, + updateGranuleIdentifiers: process.env.update_granule_identifiers === 'true', + testOverrides: { + uploadUMMGJSONCMRFileMethod: uploadEchoSpy, + metadataObjectFromCMRJSONFileMethod: () => cmrMetadata, + }, + }); + + t.is(metadataObject.GranuleUR, cmrMetadata.GranuleUR); + t.false('Identifiers' in metadataObject.DataGranule, 'Identifiers should not be exist in DataGranule'); + + process.env.update_granule_identifiers = true; + const { metadataObject: updatedMetadataObject } = await updateUMMGMetadata({ + granuleId: 'TestFixtureGranuleUR_uniq', + producerGranuleId: 'TestFixtureGranuleUR', + cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, + files: filesObject, + distEndpoint, + bucketTypes, + distributionBucketMap, + updateGranuleIdentifiers: process.env.update_granule_identifiers === 'true', + testOverrides: { + uploadUMMGJSONCMRFileMethod: uploadEchoSpy, + metadataObjectFromCMRJSONFileMethod: () => cmrMetadata, + }, + }); + console.log(`updatedMetadataObject: ${JSON.stringify(updatedMetadataObject, null, 2)}`); + t.is(updatedMetadataObject.GranuleUR, 'TestFixtureGranuleUR_uniq'); + t.is(updatedMetadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); + + process.env.update_granule_identifiers = originalUpdateGranuleIdentifiers; +}); + test.serial('updateEcho10XMLMetadata adds granule files correctly to OnlineAccessURLs/OnlineResources', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; diff --git a/tf-modules/cumulus/ingest.tf b/tf-modules/cumulus/ingest.tf index e3f85d461e5a..47735526d3aa 100644 --- a/tf-modules/cumulus/ingest.tf +++ b/tf-modules/cumulus/ingest.tf @@ -74,4 +74,7 @@ module "ingest" { # For message consumer lambdas in order to disable rule/message mismatches allow_provider_mismatch_on_rule_filter = var.allow_provider_mismatch_on_rule_filter + + # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not + update_granule_identifiers = var.default_update_granule_identifiers } diff --git a/tf-modules/cumulus/variables.tf b/tf-modules/cumulus/variables.tf index ab491c646ee8..81026b5f16db 100644 --- a/tf-modules/cumulus/variables.tf +++ b/tf-modules/cumulus/variables.tf @@ -223,6 +223,12 @@ variable "default_s3_multipart_chunksize_mb" { default = 256 } +variable "default_update_granule_identifiers" { + description = "Whether or not to automatically update granule identifiers" + type = bool + default = true +} + variable "sync_granule_s3_jitter_max_ms" { description = "Maximum random jitter in milliseconds to apply before S3 operations in SyncGranule task (0-59000). Set to 0 to disable jitter." type = number From 7c374dd8e6a14cf35ca0fd8febc644456e049bdd Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Wed, 11 Feb 2026 23:09:44 -0800 Subject: [PATCH 03/26] add file missing from last commit --- tasks/update-granules-cmr-metadata-file-links/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index d07eed11a40e..dd10ac6c5448 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -71,7 +71,7 @@ async function updateEachCmrFileMetadata( bucketTypes, cmrGranuleUrlType, distributionBucketMap, - updateGranuleIdentifiers: process.env.update_granule_identifiers, + updateGranuleIdentifiers: process.env.update_granule_identifiers === 'true', }); })); } From 0ca0245e740c2ca1a75c6c3c300bd07057c5fcbe Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 12 Feb 2026 10:11:33 -0800 Subject: [PATCH 04/26] remove test only and change to serial testing --- packages/cmrjs/tests/cmr-utils/test-cmr-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index c49615ba0418..20f1501de2a7 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -593,7 +593,7 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); }); -test.only('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is using update_granule_identifiers environment variable', async (t) => { +test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is using update_granule_identifiers environment variable', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const originalUpdateGranuleIdentifiers = process.env.update_granule_identifiers; process.env.update_granule_identifiers = false; From 353ca982ebcef668b9db947f9d8cd5e1c6ab099a Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 12 Feb 2026 10:37:45 -0800 Subject: [PATCH 05/26] * add csd-91 changelog entry. * change variable name from default_update_granule_identifiers to just update_granule_identifiers --- CHANGELOG.md | 3 +++ example/cumulus-tf/main.tf | 2 +- example/cumulus-tf/variables.tf | 2 +- tf-modules/cumulus/ingest.tf | 2 +- tf-modules/cumulus/variables.tf | 2 +- .../ingest/update-granules-cmr-metadata-file-links-task.tf | 2 +- tf-modules/ingest/variables.tf | 2 +- 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2493b5c65565..8ad85fe992a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Removed the `getUniqueGranuleByGranuleId` and `getGranuleByUniqueColumns` functions from the `@cumulus/db` package, since a single granule record can be retrieved using a unique `granule_id`. +- **CSD-91** + - Change update-granules-cmr-metadata-file-links task's updateGranuleIdentifiers flag from true to configurable environment variable `update_granule_identifiers`. + ### Notable Changes - **CUMULUS-4473** diff --git a/example/cumulus-tf/main.tf b/example/cumulus-tf/main.tf index 3c5138459b67..8a6f617c1112 100644 --- a/example/cumulus-tf/main.tf +++ b/example/cumulus-tf/main.tf @@ -137,7 +137,7 @@ module "cumulus" { sync_granule_s3_jitter_max_ms = var.sync_granule_s3_jitter_max_ms # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not - update_granule_identifiers = var.default_update_granule_identifiers + update_granule_identifiers = var.update_granule_identifiers launchpad_api = var.launchpad_api launchpad_certificate = var.launchpad_certificate diff --git a/example/cumulus-tf/variables.tf b/example/cumulus-tf/variables.tf index 608ec91294fe..ad407d59a608 100644 --- a/example/cumulus-tf/variables.tf +++ b/example/cumulus-tf/variables.tf @@ -211,7 +211,7 @@ variable "tea_distribution_url" { default = null } -variable "default_update_granule_identifiers" { +variable "update_granule_identifiers" { description = "Whether or not to automatically update granule identifiers" type = bool default = true diff --git a/tf-modules/cumulus/ingest.tf b/tf-modules/cumulus/ingest.tf index 47735526d3aa..01aac6e5e114 100644 --- a/tf-modules/cumulus/ingest.tf +++ b/tf-modules/cumulus/ingest.tf @@ -76,5 +76,5 @@ module "ingest" { allow_provider_mismatch_on_rule_filter = var.allow_provider_mismatch_on_rule_filter # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not - update_granule_identifiers = var.default_update_granule_identifiers + update_granule_identifiers = var.update_granule_identifiers } diff --git a/tf-modules/cumulus/variables.tf b/tf-modules/cumulus/variables.tf index 32a8d5decda4..fce082677002 100644 --- a/tf-modules/cumulus/variables.tf +++ b/tf-modules/cumulus/variables.tf @@ -223,7 +223,7 @@ variable "default_s3_multipart_chunksize_mb" { default = 256 } -variable "default_update_granule_identifiers" { +variable "update_granule_identifiers" { description = "Whether or not to automatically update granule identifiers" type = bool default = true diff --git a/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf b/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf index f77c8cea1b04..bd8e2713a15c 100644 --- a/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf +++ b/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf @@ -17,7 +17,7 @@ resource "aws_lambda_function" "update_granules_cmr_metadata_file_links_task" { stackName = var.prefix CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" system_bucket = var.system_bucket - update_granule_identifiers = var.default_update_granule_identifiers + update_granule_identifiers = var.update_granule_identifiers } } diff --git a/tf-modules/ingest/variables.tf b/tf-modules/ingest/variables.tf index fc8e257efdf2..170b2bc4f464 100644 --- a/tf-modules/ingest/variables.tf +++ b/tf-modules/ingest/variables.tf @@ -203,7 +203,7 @@ variable "default_log_retention_days" { description = "Optional default value that user chooses for their log retention periods" } -variable "default_update_granule_identifiers" { +variable "update_granule_identifiers" { description = "Whether or not to automatically update granule identifiers" type = bool default = true From 2624d49573924d3c4cd18a61f1465d6a7d135648 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 12 Feb 2026 13:30:56 -0800 Subject: [PATCH 06/26] restore updateGranuleIdentifiers default value for updateUMMGMetadata --- packages/cmrjs/src/cmr-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js index a6bf1e519306..e769d24d9da7 100644 --- a/packages/cmrjs/src/cmr-utils.js +++ b/packages/cmrjs/src/cmr-utils.js @@ -889,7 +889,7 @@ async function updateUMMGMetadata({ distributionBucketMap, producerGranuleId, granuleId, - updateGranuleIdentifiers, + updateGranuleIdentifiers = false, testOverrides = {}, }) { const { From 51795d033097aceb798ea811edc2b7b54fd5ca4a Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 12 Feb 2026 14:04:34 -0800 Subject: [PATCH 07/26] restore updateGranuleIdentifiers default value to updateCMRMetadata --- packages/cmrjs/src/cmr-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js index e769d24d9da7..0502360d937e 100644 --- a/packages/cmrjs/src/cmr-utils.js +++ b/packages/cmrjs/src/cmr-utils.js @@ -1223,7 +1223,7 @@ async function updateCMRMetadata({ published, bucketTypes, cmrGranuleUrlType = 'both', - updateGranuleIdentifiers, + updateGranuleIdentifiers = false, distributionBucketMap, testOverrides = {}, }) { From 7116f4c7eadd765313ccdb45a40fce20d550eb31 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Fri, 13 Feb 2026 14:05:57 -0800 Subject: [PATCH 08/26] Move update_granule_identifiers passing test from test-cmr-utils to update-granules-cmr-metadata-file-links test Add sinon and rewire to update-granules-cmr-metadata-file-links test file for testing update_granule_identifiers environment variable is been passed to updateCMRMetadata function. --- .../cmrjs/tests/cmr-utils/test-cmr-utils.js | 58 ------------------- .../tests/test-index.js | 46 +++++++++++++++ 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index 20f1501de2a7..fead58fc74cc 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -593,64 +593,6 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); }); -test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is using update_granule_identifiers environment variable', async (t) => { - const { bucketTypes, distributionBucketMap } = t.context; - const originalUpdateGranuleIdentifiers = process.env.update_granule_identifiers; - process.env.update_granule_identifiers = false; - - // Yes, ETag values always include enclosing double-quotes - const distEndpoint = 'https://distendpoint.com'; - const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); - - const cmrJSON = await fs.readFile( - path.join(__dirname, '../fixtures/MOD09GQ.A3411593.1itJ_e.006.9747594822314_v1.6.2.cmr.json'), - 'utf8' - ); - const cmrMetadata = JSON.parse(cmrJSON); - const filesObject = await readJsonFixture( - path.join(__dirname, '../fixtures/UMMGFilesObjectFixture.json') - ); - - const { metadataObject } = await updateUMMGMetadata({ - granuleId: 'TestFixtureGranuleUR_uniq', - producerGranuleId: 'TestFixtureGranuleUR', - cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, - files: filesObject, - distEndpoint, - bucketTypes, - distributionBucketMap, - updateGranuleIdentifiers: process.env.update_granule_identifiers === 'true', - testOverrides: { - uploadUMMGJSONCMRFileMethod: uploadEchoSpy, - metadataObjectFromCMRJSONFileMethod: () => cmrMetadata, - }, - }); - - t.is(metadataObject.GranuleUR, cmrMetadata.GranuleUR); - t.false('Identifiers' in metadataObject.DataGranule, 'Identifiers should not be exist in DataGranule'); - - process.env.update_granule_identifiers = true; - const { metadataObject: updatedMetadataObject } = await updateUMMGMetadata({ - granuleId: 'TestFixtureGranuleUR_uniq', - producerGranuleId: 'TestFixtureGranuleUR', - cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, - files: filesObject, - distEndpoint, - bucketTypes, - distributionBucketMap, - updateGranuleIdentifiers: process.env.update_granule_identifiers === 'true', - testOverrides: { - uploadUMMGJSONCMRFileMethod: uploadEchoSpy, - metadataObjectFromCMRJSONFileMethod: () => cmrMetadata, - }, - }); - console.log(`updatedMetadataObject: ${JSON.stringify(updatedMetadataObject, null, 2)}`); - t.is(updatedMetadataObject.GranuleUR, 'TestFixtureGranuleUR_uniq'); - t.is(updatedMetadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); - - process.env.update_granule_identifiers = originalUpdateGranuleIdentifiers; -}); - test.serial('updateEcho10XMLMetadata adds granule files correctly to OnlineAccessURLs/OnlineResources', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js index 41509f44f38e..f0dfa5dc7ce1 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js +++ b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js @@ -5,6 +5,10 @@ const path = require('path'); const test = require('ava'); const cloneDeep = require('lodash/cloneDeep'); +const sinon = require('sinon'); +const rewire = require('rewire'); +const updateGranulesCmrMetadataFileLinks = rewire('../index.js'); + const { buildS3Uri, getObject, @@ -300,3 +304,45 @@ test('updateCmrFileInfo - throws error when CMR file not found', async (t) => { message: 'CMR file not found for granule with ID granule1', }); }); + +test('Call to updateGranulesCmrMetadata should be using environment variable to set updateGranuleIdentifier flag', async (t) => { + const originalUpdateGranuleIdentifiers = process.env.update_granule_identifiers; + + const fakeFetchDistributionBucketMap = sinon.fake.resolves({}); + const restoreFetch = updateGranulesCmrMetadataFileLinks.__set__('fetchDistributionBucketMap', fakeFetchDistributionBucketMap); + + const fakeGetObjectSize = sinon.fake.resolves(123); + const restoreGetObjectSize = updateGranulesCmrMetadataFileLinks.__set__('getObjectSize', fakeGetObjectSize); + + const fakeUpdateCMRMetadata = sinon.fake.resolves({}); + const restoreUpdateCMRMetadata = updateGranulesCmrMetadataFileLinks.__set__('updateCMRMetadata', fakeUpdateCMRMetadata); + + const fakeMapFileEtags = sinon.fake.resolves({}); + const restoreMapFileEtags = updateGranulesCmrMetadataFileLinks.__set__('mapFileEtags', fakeMapFileEtags); + + const newPayload = buildPayload(t); + await validateConfig(t, newPayload.config); + await validateInput(t, newPayload.input); + const filesToUpload = cloneDeep(t.context.filesToUpload); + await uploadFiles(filesToUpload, t.context.stagingBucket); + + process.env.update_granule_identifiers = false; + await updateGranulesCmrMetadataFileLinks.updateGranulesCmrMetadata(newPayload); + + const fakeUpdateCMRMetadataCheckFalseFlag = fakeUpdateCMRMetadata.firstCall.args[0]; + t.false(fakeUpdateCMRMetadataCheckFalseFlag.updateGranuleIdentifiers, 'updateGranuleIdentifiers should be false when the environment variable is set to false'); + + process.env.update_granule_identifiers = true; + fakeUpdateCMRMetadata.resetHistory(); + await updateGranulesCmrMetadataFileLinks.updateGranulesCmrMetadata(newPayload); + const fakeUpdateCMRMetadataCheckTrueFlag = fakeUpdateCMRMetadata.firstCall.args[0]; + t.true(fakeUpdateCMRMetadataCheckTrueFlag.updateGranuleIdentifiers, 'updateGranuleIdentifiers should be true when the environment variable is set to true'); + + restoreFetch(); + restoreGetObjectSize(); + restoreMapFileEtags(); + restoreUpdateCMRMetadata(); + sinon.restore(); + + process.env.update_granule_identifiers = originalUpdateGranuleIdentifiers; +}); From a7adf255e02cd5a53d401d5b158c9903ba5ef1fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Feb 2026 22:12:58 +0000 Subject: [PATCH 09/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tf-modules/cumulus/ingest.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf-modules/cumulus/ingest.tf b/tf-modules/cumulus/ingest.tf index 153de25591c1..f9ca53b4963d 100644 --- a/tf-modules/cumulus/ingest.tf +++ b/tf-modules/cumulus/ingest.tf @@ -77,6 +77,6 @@ module "ingest" { # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not update_granule_identifiers = var.update_granule_identifiers - + private_api_lambda_arn = module.archive.private_api_lambda_arn } From 50320e7225e7e2c089857df2b2db3810bdb34740 Mon Sep 17 00:00:00 2001 From: Rohan Weeden Date: Mon, 16 Feb 2026 18:36:05 -0500 Subject: [PATCH 10/26] CUMULUS-4395 CnmResponse terraform (#4239) * Add terraform * Update workflow deployed CnmResponse lambda to avoid name conflict * Update CHANGELOG.md * Update granule-invalidator task terraform to match cnm-response * Update cnm-to-cma task to match other corified tasks * Rework packaging to use global package_uv.sh --- CHANGELOG.md | 2 + example/cumulus-tf/cnm_response_task.tf | 12 +- tasks/cnm-response/bin/package.sh | 29 -- tasks/cnm-response/deploy/main.tf | 33 ++ tasks/cnm-response/deploy/output.tf | 9 + tasks/cnm-response/deploy/variables.tf | 41 ++ tasks/cnm-response/deploy/versions.tf | 10 + tasks/cnm-response/package.json | 2 +- tasks/cnm-to-cma/README.md | 47 +- tasks/cnm-to-cma/deploy/lambda.tf | 24 - tasks/cnm-to-cma/deploy/main.tf | 38 +- tasks/cnm-to-cma/deploy/output.tf | 20 +- tasks/cnm-to-cma/deploy/variable.tf | 41 -- tasks/cnm-to-cma/deploy/variables.tf | 41 ++ tasks/cnm-to-cma/deploy/versions.tf | 10 + tasks/cnm-to-cma/package.json | 4 +- tasks/cnm-to-cma/pyproject.toml | 19 +- tasks/cnm-to-cma/ruff.toml | 4 +- tasks/cnm-to-cma/src/cnm_to_cma/__init__.py | 0 .../src/{cnm2cma => cnm_to_cma}/cnm_to_cma.py | 7 +- .../src/{cnm2cma => cnm_to_cma}/models_cnm.py | 2 +- .../{cnm2cma => cnm_to_cma}/models_granule.py | 0 .../schemas/config.json | 0 .../schemas/input.json | 0 .../schemas/output.json | 4 +- tasks/cnm-to-cma/tests/test_cnm_to_cma.py | 5 +- tasks/cnm-to-cma/uv.lock | 443 ++++++++++++++++-- tasks/granule-invalidator/deploy/outputs.tf | 22 +- tf-modules/cumulus/outputs.tf | 4 + tf-modules/ingest/cnm-response.tf | 12 + tf-modules/ingest/outputs.tf | 13 +- 31 files changed, 675 insertions(+), 223 deletions(-) delete mode 100755 tasks/cnm-response/bin/package.sh create mode 100644 tasks/cnm-response/deploy/main.tf create mode 100644 tasks/cnm-response/deploy/output.tf create mode 100644 tasks/cnm-response/deploy/variables.tf create mode 100644 tasks/cnm-response/deploy/versions.tf delete mode 100644 tasks/cnm-to-cma/deploy/lambda.tf delete mode 100644 tasks/cnm-to-cma/deploy/variable.tf create mode 100644 tasks/cnm-to-cma/deploy/variables.tf create mode 100644 tasks/cnm-to-cma/deploy/versions.tf create mode 100644 tasks/cnm-to-cma/src/cnm_to_cma/__init__.py rename tasks/cnm-to-cma/src/{cnm2cma => cnm_to_cma}/cnm_to_cma.py (96%) rename tasks/cnm-to-cma/src/{cnm2cma => cnm_to_cma}/models_cnm.py (99%) rename tasks/cnm-to-cma/src/{cnm2cma => cnm_to_cma}/models_granule.py (100%) rename tasks/cnm-to-cma/src/{cnm2cma => cnm_to_cma}/schemas/config.json (100%) rename tasks/cnm-to-cma/src/{cnm2cma => cnm_to_cma}/schemas/input.json (100%) rename tasks/cnm-to-cma/src/{cnm2cma => cnm_to_cma}/schemas/output.json (93%) create mode 100644 tf-modules/ingest/cnm-response.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index b5de759a9f11..0480b49d49e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added supporting Terraform for the granule-invalidator task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus. - **CUMULUS-4394** - Added python code for CnmResponse task adapted from https://github.com/podaac/cumulus-cnm-response-task +- **CUMULUS-4395** + - Added supporting Terraform for the CnmResponse task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus. - **CUMULUS-4498** - Added `states:StartExecution` action to the `-steprole` IAM role. diff --git a/example/cumulus-tf/cnm_response_task.tf b/example/cumulus-tf/cnm_response_task.tf index 5aa34e996ac6..182eabae8edc 100644 --- a/example/cumulus-tf/cnm_response_task.tf +++ b/example/cumulus-tf/cnm_response_task.tf @@ -12,7 +12,7 @@ resource "null_resource" "get_cnmResponse" { } } -resource aws_s3_bucket_object "cnm_response_lambda_zip" { +resource "aws_s3_object" "cnm_response_lambda_zip" { depends_on = [null_resource.get_cnmResponse] bucket = var.system_bucket key = "${var.prefix}/${local.cnm_response_filename}" @@ -20,16 +20,16 @@ resource aws_s3_bucket_object "cnm_response_lambda_zip" { } resource "aws_lambda_function" "cnm_response_task" { - depends_on = [aws_s3_bucket_object.cnm_response_lambda_zip] - function_name = "${var.prefix}-CnmResponse" + depends_on = [aws_s3_object.cnm_response_lambda_zip] + function_name = "${var.prefix}-CnmResponseJava" s3_bucket = var.system_bucket - s3_key = aws_s3_bucket_object.cnm_response_lambda_zip.id + s3_key = aws_s3_object.cnm_response_lambda_zip.id handler = "gov.nasa.cumulus.CNMResponse::handleRequestStreams" role = module.cumulus.lambda_processing_role_arn runtime = "java21" timeout = 300 memory_size = 256 - source_code_hash = aws_s3_bucket_object.cnm_response_lambda_zip.etag + source_code_hash = aws_s3_object.cnm_response_lambda_zip.etag layers = [var.cumulus_message_adapter_lambda_layer_version_arn] @@ -43,7 +43,7 @@ resource "aws_lambda_function" "cnm_response_task" { dynamic "vpc_config" { for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] content { - subnet_ids = var.lambda_subnet_ids + subnet_ids = var.lambda_subnet_ids security_group_ids = [aws_security_group.no_ingress_all_egress.id] } } diff --git a/tasks/cnm-response/bin/package.sh b/tasks/cnm-response/bin/package.sh deleted file mode 100755 index 1de06ec79764..000000000000 --- a/tasks/cnm-response/bin/package.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -DIR=$1 -# find packages location -echo "Entering $DIR" -echo $PWD -cd "$DIR" || exit 1 - -# ensure packages are up to date using uv -uv sync --no-dev --frozen - -# package dependencies -SITE_PACKAGES=$(find "$DIR"/.venv/lib/python*/site-packages -type d | head -1) -echo "Entering $SITE_PACKAGES" -cd "$SITE_PACKAGES" || exit 1 -cp -R ./* "$DIR/dist/" - -cd "$DIR" || exit 1 - -cp -R ./src/* schemas/ ./dist/ - -cd ./dist || exit 1 - -node ../../../bin/zip.js lambda.zip $(ls | grep -v lambda.zip) - -cd .. || exit 1 - -# Re-sync dev dependencies for any post-packaging tasks -uv sync diff --git a/tasks/cnm-response/deploy/main.tf b/tasks/cnm-response/deploy/main.tf new file mode 100644 index 000000000000..f7a5ad1b0d66 --- /dev/null +++ b/tasks/cnm-response/deploy/main.tf @@ -0,0 +1,33 @@ +resource "aws_lambda_function" "cnm_response_task" { + function_name = "${var.prefix}-CnmResponse" + filename = "${path.module}/../dist/final/lambda.zip" + source_code_hash = filebase64sha256("${path.module}/../dist/final/lambda.zip") + handler = "cnm_response.lambda_handler" + role = var.lambda_processing_role_arn + runtime = "python3.12" + timeout = var.lambda_timeout + memory_size = var.lambda_memory_size + + environment { + variables = { + stackName = var.prefix + CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" + } + } + + dynamic "vpc_config" { + for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] + content { + subnet_ids = var.lambda_subnet_ids + security_group_ids = [var.security_group_id] + } + } + + tags = var.tags +} + +resource "aws_cloudwatch_log_group" "cnm_response_task" { + name = "/aws/lambda/${aws_lambda_function.cnm_response_task.function_name}" + retention_in_days = var.default_log_retention_days + tags = var.tags +} diff --git a/tasks/cnm-response/deploy/output.tf b/tasks/cnm-response/deploy/output.tf new file mode 100644 index 000000000000..cfd61e014459 --- /dev/null +++ b/tasks/cnm-response/deploy/output.tf @@ -0,0 +1,9 @@ +output "lambda_function" { + description = "The task lambda function" + value = aws_lambda_function.cnm_response_task +} + +output "log_group" { + description = "The task lambda function log group" + value = aws_cloudwatch_log_group.cnm_response_task +} diff --git a/tasks/cnm-response/deploy/variables.tf b/tasks/cnm-response/deploy/variables.tf new file mode 100644 index 000000000000..f32dcc54cdfd --- /dev/null +++ b/tasks/cnm-response/deploy/variables.tf @@ -0,0 +1,41 @@ +variable "default_log_retention_days" { + description = "The number of days to retain logs in CloudWatch" + type = number +} + +variable "lambda_processing_role_arn" { + type = string +} + +variable "lambda_subnet_ids" { + type = list(string) + default = [] +} + +variable "lambda_timeout" { + description = "Timeout value for the Lambda function in seconds" + type = number + default = 300 +} + +variable "lambda_memory_size" { + description = "Memory size for the Lambda function in MB" + type = number + default = 512 +} + +variable "prefix" { + type = string +} + +variable "security_group_id" { + description = "Security group ID for Lambda VPC configuration" + type = string + default = "" +} + +variable "tags" { + description = "Tags to be applied to managed resources" + type = map(string) + default = {} +} diff --git a/tasks/cnm-response/deploy/versions.tf b/tasks/cnm-response/deploy/versions.tf new file mode 100644 index 000000000000..ddfcb0e054b9 --- /dev/null +++ b/tasks/cnm-response/deploy/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} diff --git a/tasks/cnm-response/package.json b/tasks/cnm-response/package.json index 5eb0fa4e3ffb..a71bf41be538 100644 --- a/tasks/cnm-response/package.json +++ b/tasks/cnm-response/package.json @@ -20,7 +20,7 @@ "clean": "rm -rf dist && rm -rf .venv && mkdir dist", "generate-task-schemas": "npx generate-task-schemas . files", "lint": "uv run ruff check . && uv run ruff format --diff . && uv run mypy src/cnm_response", - "package": "npm run clean && ./bin/package.sh $(pwd)", + "package": "npm run clean && ../../bin/package_uv.sh 3.12 ${PWD}", "prepare": "npm run build", "test": "uv run pytest -q" }, diff --git a/tasks/cnm-to-cma/README.md b/tasks/cnm-to-cma/README.md index 7de624eb208f..8201c6108e42 100644 --- a/tasks/cnm-to-cma/README.md +++ b/tasks/cnm-to-cma/README.md @@ -5,6 +5,7 @@ format. CNM schema is defined in schemas/cumulus_sns_schema.json payload file schema defined in schemas/files.scheam.json + ## Message configuration For more information on configuring a Cumulus Message Adapter task, see [the Cumulus workflow input/output documentation](https://nasa.github.io/cumulus/docs/workflows/input_output). @@ -13,10 +14,9 @@ For more information on configuring a Cumulus Message Adapter task, see [the Cum Config object fields: -| field name | type | default | description -|---------------------|--------|------------| ----------- -| collection | object | (optional) | collection.granuleIdExtraction:Regex used to extract granuleId from filenames - +| field name | type | default | description | +| ---------- | ------ | ---------- | ----------------------------------------------------------------------------- | +| collection | object | (optional) | collection.granuleIdExtraction:Regex used to extract granuleId from filenames | ```angular2html "TranslateMessage": { @@ -63,6 +63,7 @@ Config object fields: "Next": "Report" }, ``` + The Step Function task definition utilizes a task_config object to manage dynamic inputs. Within this configuration, the collection field is mapped using JSONPath to the metadata: Config Input: "collection": "{$.meta.collection}" @@ -73,6 +74,7 @@ Users can define a regular expression via collection.granuleIdExtraction. This r Terraform Implementation This task is designed as a modular component. You can load it into your infrastructure using the following Terraform example: + ```angular2html module "cnm_to_cma_module" { @@ -97,12 +99,11 @@ module "cnm_to_cma_module" { ### Input - Input array specification: -| field name | type | default | description -| ---------- |--------| ------- | ----------- -| N/A | object | (required) | cnm message +| field name | type | default | description | +| ---------- | ------ | ---------- | ----------- | +| N/A | object | (required) | cnm message | the lambda's event.get('input') is the cnm message. @@ -110,11 +111,10 @@ the lambda's event.get('input') is the cnm message. Output object fields: -| field name | type | default | description -|------------|-----------------| ------- | ----------- -| granules | array\ | N/A | cma payload with list of cma files -| cnm | object | N/A | the original cnm message. - +| field name | type | default | description | +| ---------- | --------------- | ------- | ---------------------------------- | +| granules | array\ | N/A | cma payload with list of cma files | +| cnm | object | N/A | the original cnm message. | Data Mapping and Payload Configuration @@ -123,10 +123,13 @@ When configuring the task, please refer to the example provided in the "Config" Granule Mapping: The output_granules field must be mapped directly to the primary payload. Metadata Attachment: The original Cloud Notification Mechanism (CNM) message should be nested under the meta object. + ### Example workflow configuration and use + The output key : output_granules should be mapped to the payload key in the workflow's output. Workflow developer could choose to the original cnm message to be stored in meta.cnm key. Example workflow: + ```angular2html "TranslateMessage": { "Parameters": { @@ -172,7 +175,9 @@ Example workflow: "Next": "Report" }, ``` + ## Architecture + ```mermaid architecture-cnm-2-cma group trigger(cloud) [Starting Task] @@ -182,20 +187,25 @@ architecture-cnm-2-cma ``` ## Internal Dependencies + Python cumulus-message-adapter library ### External Dependencies + None ## Development and Deployment + #### Developer Notes + - Build : Temporarily use build_lambda.sh which depends on poetry to build the lambda.zip file. --- poetry self add poetry-plugin-export to enable poetry export. --- alternatively, download pydantic_core wheel from here : https://pypi.org/project/pydantic_core/#files by -- using the manylinux wheel with python version close to this project's + -- poetry self add poetry-plugin-export to enable poetry export. + -- alternatively, download pydantic_core wheel from here : https://pypi.org/project/pydantic_core/#files by +- using the manylinux wheel with python version close to this project's - About json schema compiling to Plaint Python classes: - - used datamodel-code-generator tool: https://koxudaxi.github.io/datamodel-code-generator/ to generate pydantic models from json schema files. - - example below + - used datamodel-code-generator tool: https://koxudaxi.github.io/datamodel-code-generator/ to generate pydantic models from json schema files. + - example below + ```angular2html pip install datamodel-code-generator # Or with HTTP support for remote references @@ -212,7 +222,6 @@ datamodel-codegen \ To make a contribution, please [see our Cumulus contributing guidelines](https://github.com/nasa/cumulus/blob/master/CONTRIBUTING.md) and our documentation on [adding a task](https://nasa.github.io/cumulus/docs/adding-a-task) - ## About Cumulus Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams. diff --git a/tasks/cnm-to-cma/deploy/lambda.tf b/tasks/cnm-to-cma/deploy/lambda.tf deleted file mode 100644 index fdb19ccc1bad..000000000000 --- a/tasks/cnm-to-cma/deploy/lambda.tf +++ /dev/null @@ -1,24 +0,0 @@ -resource "aws_lambda_function" "cnm_to_cma" { - filename = "${path.module}/cnm2cma.zip" - function_name = "${var.prefix}-CNMToCMA" - source_code_hash = filebase64sha256("${path.module}/cnm2cma.zip") - handler = "cnm2cma.cnm_to_cma.handler" - role = var.lambda_role - runtime = "python3.13" - timeout = var.timeout - memory_size = var.memory_size - architectures = ["arm64"] - - vpc_config { - subnet_ids = var.subnet_ids - security_group_ids = var.security_group_ids - } - - tags = local.tags -} - -resource "aws_cloudwatch_log_group" "cnm_to_cma" { - name = "/aws/lambda/${aws_lambda_function.cnm_to_cma.function_name}" - retention_in_days = var.default_log_retention_days - tags = local.tags -} diff --git a/tasks/cnm-to-cma/deploy/main.tf b/tasks/cnm-to-cma/deploy/main.tf index 848c1b647807..ac216f0cdf77 100644 --- a/tasks/cnm-to-cma/deploy/main.tf +++ b/tasks/cnm-to-cma/deploy/main.tf @@ -1,7 +1,33 @@ -locals { - environment = var.prefix - tags = merge(var.default_tags, { - team: "Cumulus Coreification Engineering", - application: var.app_name - }) +resource "aws_lambda_function" "cnm_to_cma_task" { + function_name = "${var.prefix}-CNMToCMA" + filename = "${path.module}/../dist/lambda.zip" + source_code_hash = filebase64sha256("${path.module}/../dist/lambda.zip") + handler = "cnm_to_cma.cnm_to_cma.handler" + role = var.lambda_processing_role_arn + runtime = "python3.12" + timeout = var.lambda_timeout + memory_size = var.lambda_memory_size + + environment { + variables = { + stackName = var.prefix + CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" + } + } + + dynamic "vpc_config" { + for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] + content { + subnet_ids = var.lambda_subnet_ids + security_group_ids = [var.security_group_id] + } + } + + tags = var.tags +} + +resource "aws_cloudwatch_log_group" "cnm_to_cma_task" { + name = "/aws/lambda/${aws_lambda_function.cnm_to_cma_task.function_name}" + retention_in_days = var.default_log_retention_days + tags = var.tags } diff --git a/tasks/cnm-to-cma/deploy/output.tf b/tasks/cnm-to-cma/deploy/output.tf index b3599c9aef56..a3747a9fafcf 100644 --- a/tasks/cnm-to-cma/deploy/output.tf +++ b/tasks/cnm-to-cma/deploy/output.tf @@ -3,21 +3,7 @@ output "lambda_function" { value = aws_lambda_function.cnm_to_cma } -output "lambda_function_name" { - description = "Name of the Lambda function" - value = aws_lambda_function.cnm_to_cma.function_name -} - -output "lambda_function_invoke_arn" { - description = "Invoke ARN of the Lambda function" - value = aws_lambda_function.cnm_to_cma.invoke_arn -} - -output "lambda_function_last_modified" { - description = "Last modified date of the Lambda function" - value = aws_lambda_function.cnm_to_cma.last_modified -} -output "lambda_log_group_name" { - description = "Name of the Lambda's CloudWatch log group" - value = aws_cloudwatch_log_group.cnm_to_cma.name +output "log_group" { + description = "The task lambda function log group" + value = aws_cloudwatch_log_group.cnm_to_cma } diff --git a/tasks/cnm-to-cma/deploy/variable.tf b/tasks/cnm-to-cma/deploy/variable.tf deleted file mode 100644 index 4efd4effac41..000000000000 --- a/tasks/cnm-to-cma/deploy/variable.tf +++ /dev/null @@ -1,41 +0,0 @@ -variable "prefix" { - type = string -} - -# The ARN of the IAM role that the Lambda function will assume when it executes. -# This role must have the necessary permissions to perform its tasks. -variable "lambda_role" { - type = string -} - -variable "security_group_ids" { - type = list(string) -} - -variable "subnet_ids" { - type = list(string) -} - -variable "app_name" { - default = "cnm_to_cma" -} - -variable "default_tags" { - type = map(string) - default = {} -} - -variable "memory_size" { - type = number - default = 512 -} - -variable "timeout" { - type = number - default = 120 -} - -variable "default_log_retention_days" { - description = "The number of days to retain logs in CloudWatch" - type = number -} diff --git a/tasks/cnm-to-cma/deploy/variables.tf b/tasks/cnm-to-cma/deploy/variables.tf new file mode 100644 index 000000000000..f32dcc54cdfd --- /dev/null +++ b/tasks/cnm-to-cma/deploy/variables.tf @@ -0,0 +1,41 @@ +variable "default_log_retention_days" { + description = "The number of days to retain logs in CloudWatch" + type = number +} + +variable "lambda_processing_role_arn" { + type = string +} + +variable "lambda_subnet_ids" { + type = list(string) + default = [] +} + +variable "lambda_timeout" { + description = "Timeout value for the Lambda function in seconds" + type = number + default = 300 +} + +variable "lambda_memory_size" { + description = "Memory size for the Lambda function in MB" + type = number + default = 512 +} + +variable "prefix" { + type = string +} + +variable "security_group_id" { + description = "Security group ID for Lambda VPC configuration" + type = string + default = "" +} + +variable "tags" { + description = "Tags to be applied to managed resources" + type = map(string) + default = {} +} diff --git a/tasks/cnm-to-cma/deploy/versions.tf b/tasks/cnm-to-cma/deploy/versions.tf new file mode 100644 index 000000000000..ddfcb0e054b9 --- /dev/null +++ b/tasks/cnm-to-cma/deploy/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} diff --git a/tasks/cnm-to-cma/package.json b/tasks/cnm-to-cma/package.json index 2d35ed3175b2..e01f6d3eaeda 100644 --- a/tasks/cnm-to-cma/package.json +++ b/tasks/cnm-to-cma/package.json @@ -19,8 +19,8 @@ "build": "rm -rf dist && mkdir dist && npm run generate-task-schemas && cp -R schemas dist/", "clean": "rm -rf dist && rm -rf .venv && mkdir dist", "generate-task-schemas": "npx generate-task-schemas . files", - "lint": "uv run ruff check . && uv run ruff format --diff . && uv run mypy src/cnm2cma", - "package": "npm run clean && ../../bin/package_uv.sh 3.13 ${PWD}", + "lint": "uv run ruff check . && uv run ruff format --diff . && uv run mypy src/cnm_to_cma", + "package": "npm run clean && ../../bin/package_uv.sh 3.12 ${PWD}", "prepare": "npm run build", "test": "uv run pytest -q" }, diff --git a/tasks/cnm-to-cma/pyproject.toml b/tasks/cnm-to-cma/pyproject.toml index f50f697806bd..81b8fbf3f40e 100644 --- a/tasks/cnm-to-cma/pyproject.toml +++ b/tasks/cnm-to-cma/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "cnm-to-cma" -version = "0.1.0" +version = "21.0.1" description = "Translate CNM message to CMA" authors = [ { name = "ESHINDavid Yen", email = "david.yen@jpl.nasa.gov" } ] readme = "README.md" -requires-python = ">=3.12,<4.0" +requires-python = "~=3.12.0" dependencies = [ "pydantic (>=2.12.5,<3.0.0)", @@ -16,12 +16,21 @@ dependencies = [ [dependency-groups] dev = [ - "pytest (>=9.0.2,<10.0.0)" + "freezegun~=1.5", + "moto~=5.1", + "mypy~=1.19", + "pytest~=9.0", + "pytest-cov~=7.0", + "pytest-mock~=3.15", + "ruff~=0.14.0", ] [tool.pytest] -packages = [{ include = "cnm2cma", from = "src" }] - +minversion = "9.0" +addopts = [ + "--cov=cnm_to_cma", + "--cov-report=lcov:.nyc_output", +] pythonpath = ["src"] testpaths = [ "tests", diff --git a/tasks/cnm-to-cma/ruff.toml b/tasks/cnm-to-cma/ruff.toml index 0ee889f7084a..0956e2b64caa 100644 --- a/tasks/cnm-to-cma/ruff.toml +++ b/tasks/cnm-to-cma/ruff.toml @@ -1,4 +1,4 @@ exclude = [ - "src/cnm2cma/models_cnm.py", # generated code, skip linting - "src/cnm2cma/models_granule.py" # generated code, skip linting + "src/cnm_to_cma/models_cnm.py", # generated code, skip linting + "src/cnm_to_cma/models_granule.py" # generated code, skip linting ] diff --git a/tasks/cnm-to-cma/src/cnm_to_cma/__init__.py b/tasks/cnm-to-cma/src/cnm_to_cma/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tasks/cnm-to-cma/src/cnm2cma/cnm_to_cma.py b/tasks/cnm-to-cma/src/cnm_to_cma/cnm_to_cma.py similarity index 96% rename from tasks/cnm-to-cma/src/cnm2cma/cnm_to_cma.py rename to tasks/cnm-to-cma/src/cnm_to_cma/cnm_to_cma.py index e5688d4814c3..a2f579e8da7d 100644 --- a/tasks/cnm-to-cma/src/cnm2cma/cnm_to_cma.py +++ b/tasks/cnm-to-cma/src/cnm_to_cma/cnm_to_cma.py @@ -5,15 +5,16 @@ from typing import Any import pydantic -from cnm2cma import models_cnm, models_granule from cumulus_logger import CumulusLogger from run_cumulus_task import run_cumulus_task +from . import models_cnm, models_granule + # Create Cumulus Logger instance LOGGER = CumulusLogger("cnm_to_cma") -def task(event: dict[str, list[str] | dict], context: object) -> dict[str, Any]: +def task(event: dict[str, Any], context: object) -> dict[str, Any]: """Entry point of the lambda Args: event: Passed through from {handler} @@ -71,7 +72,7 @@ def mapper(cnm: dict, config: dict) -> models_granule.Granule: ) LOGGER.info(f"Granule ID: {granule_id}") cnm_input_files: list[models_cnm.File] = get_cnm_input_files(product) - cma_files: list[models_cnm.File] = create_granule_files(cnm_input_files) + cma_files: list[models_granule.File] = create_granule_files(cnm_input_files) granule = models_granule.Granule( granuleId=granule_id, producerGranuleId=cnm_model.root.product.producerGranuleId, diff --git a/tasks/cnm-to-cma/src/cnm2cma/models_cnm.py b/tasks/cnm-to-cma/src/cnm_to_cma/models_cnm.py similarity index 99% rename from tasks/cnm-to-cma/src/cnm2cma/models_cnm.py rename to tasks/cnm-to-cma/src/cnm_to_cma/models_cnm.py index db664bf1ce30..d7d2a0b23f05 100644 --- a/tasks/cnm-to-cma/src/cnm2cma/models_cnm.py +++ b/tasks/cnm-to-cma/src/cnm_to_cma/models_cnm.py @@ -255,7 +255,7 @@ class CloudNotificationMessageCnm122(BaseModel): ..., description="The response message type sent. Can be a success message or error message. Akin to both the PAN and PDRD", ) - product: Product2 | Product3 | None = None + product: Product2 | Product3 class CloudNotificationMessageCnm12( diff --git a/tasks/cnm-to-cma/src/cnm2cma/models_granule.py b/tasks/cnm-to-cma/src/cnm_to_cma/models_granule.py similarity index 100% rename from tasks/cnm-to-cma/src/cnm2cma/models_granule.py rename to tasks/cnm-to-cma/src/cnm_to_cma/models_granule.py diff --git a/tasks/cnm-to-cma/src/cnm2cma/schemas/config.json b/tasks/cnm-to-cma/src/cnm_to_cma/schemas/config.json similarity index 100% rename from tasks/cnm-to-cma/src/cnm2cma/schemas/config.json rename to tasks/cnm-to-cma/src/cnm_to_cma/schemas/config.json diff --git a/tasks/cnm-to-cma/src/cnm2cma/schemas/input.json b/tasks/cnm-to-cma/src/cnm_to_cma/schemas/input.json similarity index 100% rename from tasks/cnm-to-cma/src/cnm2cma/schemas/input.json rename to tasks/cnm-to-cma/src/cnm_to_cma/schemas/input.json diff --git a/tasks/cnm-to-cma/src/cnm2cma/schemas/output.json b/tasks/cnm-to-cma/src/cnm_to_cma/schemas/output.json similarity index 93% rename from tasks/cnm-to-cma/src/cnm2cma/schemas/output.json rename to tasks/cnm-to-cma/src/cnm_to_cma/schemas/output.json index e5f4730a565a..8bd465f2b825 100644 --- a/tasks/cnm-to-cma/src/cnm2cma/schemas/output.json +++ b/tasks/cnm-to-cma/src/cnm_to_cma/schemas/output.json @@ -1,6 +1,6 @@ { - "title": "SyncGranuleInput", - "description": "Describes the input expected by the sync-granule task", + "title": "CNMToCMAInput", + "description": "Describes the input expected by the cnm-to-cma task", "type": "object", "required": ["granules"], "properties": { diff --git a/tasks/cnm-to-cma/tests/test_cnm_to_cma.py b/tasks/cnm-to-cma/tests/test_cnm_to_cma.py index 23d913f749b0..95ef5aebe2ee 100644 --- a/tasks/cnm-to-cma/tests/test_cnm_to_cma.py +++ b/tasks/cnm-to-cma/tests/test_cnm_to_cma.py @@ -2,8 +2,9 @@ import pydantic import pytest -from cnm2cma import models_cnm, models_granule -from cnm2cma.cnm_to_cma import create_granule_files, get_cnm_input_files, mapper + +from cnm_to_cma import models_cnm, models_granule +from cnm_to_cma.cnm_to_cma import create_granule_files, get_cnm_input_files, mapper class TestCNMToCMA: diff --git a/tasks/cnm-to-cma/uv.lock b/tasks/cnm-to-cma/uv.lock index 5fde70ccafe9..2c4e263cfaa0 100644 --- a/tasks/cnm-to-cma/uv.lock +++ b/tasks/cnm-to-cma/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.12, <4.0" +requires-python = "==3.12.*" [[package]] name = "annotated-types" @@ -48,9 +48,66 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/6c/522e05388aa6fc66cf8ea46c6b29809a1a6f527ea864998b01ffb368ca36/botocore-1.40.76-py3-none-any.whl", hash = "sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4", size = 14161738, upload-time = "2025-11-18T20:22:55.332Z" }, ] +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + [[package]] name = "cnm-to-cma" -version = "0.1.0" +version = "21.0.1" source = { virtual = "." } dependencies = [ { name = "cumulus-message-adapter-python" }, @@ -60,7 +117,13 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "freezegun" }, + { name = "moto" }, + { name = "mypy" }, { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "ruff" }, ] [package.metadata] @@ -71,7 +134,15 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=9.0.2,<10.0.0" }] +dev = [ + { name = "freezegun", specifier = "~=1.5" }, + { name = "moto", specifier = "~=5.1" }, + { name = "mypy", specifier = "~=1.19" }, + { name = "pytest", specifier = "~=9.0" }, + { name = "pytest-cov", specifier = "~=7.0" }, + { name = "pytest-mock", specifier = "~=3.15" }, + { name = "ruff", specifier = "~=0.14.0" }, +] [[package]] name = "colorama" @@ -82,6 +153,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coverage" +version = "7.13.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/56/95b7e30fa389756cb56630faa728da46a27b8c6eb46f9d557c68fff12b65/coverage-7.13.4.tar.gz", hash = "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", size = 827239, upload-time = "2026-02-09T12:59:03.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/81/4ce2fdd909c5a0ed1f6dedb88aa57ab79b6d1fbd9b588c1ac7ef45659566/coverage-7.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459", size = 219449, upload-time = "2026-02-09T12:56:54.889Z" }, + { url = "https://files.pythonhosted.org/packages/5d/96/5238b1efc5922ddbdc9b0db9243152c09777804fb7c02ad1741eb18a11c0/coverage-7.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40aa8808140e55dc022b15d8aa7f651b6b3d68b365ea0398f1441e0b04d859c3", size = 219810, upload-time = "2026-02-09T12:56:56.33Z" }, + { url = "https://files.pythonhosted.org/packages/78/72/2f372b726d433c9c35e56377cf1d513b4c16fe51841060d826b95caacec1/coverage-7.13.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5b856a8ccf749480024ff3bd7310adaef57bf31fd17e1bfc404b7940b6986634", size = 251308, upload-time = "2026-02-09T12:56:57.858Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/2ea570925524ef4e00bb6c82649f5682a77fac5ab910a65c9284de422600/coverage-7.13.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c048ea43875fbf8b45d476ad79f179809c590ec7b79e2035c662e7afa3192e3", size = 254052, upload-time = "2026-02-09T12:56:59.754Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ac/45dc2e19a1939098d783c846e130b8f862fbb50d09e0af663988f2f21973/coverage-7.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7b38448866e83176e28086674fe7368ab8590e4610fb662b44e345b86d63ffa", size = 255165, upload-time = "2026-02-09T12:57:01.287Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4d/26d236ff35abc3b5e63540d3386e4c3b192168c1d96da5cb2f43c640970f/coverage-7.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de6defc1c9badbf8b9e67ae90fd00519186d6ab64e5cc5f3d21359c2a9b2c1d3", size = 257432, upload-time = "2026-02-09T12:57:02.637Z" }, + { url = "https://files.pythonhosted.org/packages/ec/55/14a966c757d1348b2e19caf699415a2a4c4f7feaa4bbc6326a51f5c7dd1b/coverage-7.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7eda778067ad7ffccd23ecffce537dface96212576a07924cbf0d8799d2ded5a", size = 251716, upload-time = "2026-02-09T12:57:04.056Z" }, + { url = "https://files.pythonhosted.org/packages/77/33/50116647905837c66d28b2af1321b845d5f5d19be9655cb84d4a0ea806b4/coverage-7.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e87f6c587c3f34356c3759f0420693e35e7eb0e2e41e4c011cb6ec6ecbbf1db7", size = 253089, upload-time = "2026-02-09T12:57:05.503Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/8efb11a46e3665d92635a56e4f2d4529de6d33f2cb38afd47d779d15fc99/coverage-7.13.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8248977c2e33aecb2ced42fef99f2d319e9904a36e55a8a68b69207fb7e43edc", size = 251232, upload-time = "2026-02-09T12:57:06.879Z" }, + { url = "https://files.pythonhosted.org/packages/51/24/8cd73dd399b812cc76bb0ac260e671c4163093441847ffe058ac9fda1e32/coverage-7.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:25381386e80ae727608e662474db537d4df1ecd42379b5ba33c84633a2b36d47", size = 255299, upload-time = "2026-02-09T12:57:08.245Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/0a4b12f1d0e029ce1ccc1c800944a9984cbe7d678e470bb6d3c6bc38a0da/coverage-7.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ee756f00726693e5ba94d6df2bdfd64d4852d23b09bb0bc700e3b30e6f333985", size = 250796, upload-time = "2026-02-09T12:57:10.142Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/6002fbf88f6698ca034360ce474c406be6d5a985b3fdb3401128031eef6b/coverage-7.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0", size = 252673, upload-time = "2026-02-09T12:57:12.197Z" }, + { url = "https://files.pythonhosted.org/packages/de/c6/a0279f7c00e786be75a749a5674e6fa267bcbd8209cd10c9a450c655dfa7/coverage-7.13.4-cp312-cp312-win32.whl", hash = "sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246", size = 221990, upload-time = "2026-02-09T12:57:14.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/4e/c0a25a425fcf5557d9abd18419c95b63922e897bc86c1f327f155ef234a9/coverage-7.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:9401ebc7ef522f01d01d45532c68c5ac40fb27113019b6b7d8b208f6e9baa126", size = 222800, upload-time = "2026-02-09T12:57:15.944Z" }, + { url = "https://files.pythonhosted.org/packages/47/ac/92da44ad9a6f4e3a7debd178949d6f3769bedca33830ce9b1dcdab589a37/coverage-7.13.4-cp312-cp312-win_arm64.whl", hash = "sha256:b1ec7b6b6e93255f952e27ab58fbc68dcc468844b16ecbee881aeb29b6ab4d8d", size = 221415, upload-time = "2026-02-09T12:57:17.497Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, +] + +[[package]] +name = "cryptography" +version = "46.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" }, + { url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" }, + { url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" }, + { url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" }, + { url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" }, + { url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" }, + { url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" }, + { url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" }, + { url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" }, + { url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" }, + { url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" }, + { url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" }, + { url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" }, + { url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" }, + { url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" }, + { url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" }, + { url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" }, + { url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" }, +] + [[package]] name = "cumulus-message-adapter" version = "2.0.5" @@ -102,6 +236,27 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/4a/92/8e97cc640181f72eb7b2fd6c0524b9c28720c5c06da264ed8e8fa0704ecf/cumulus_message_adapter_python-2.4.0.tar.gz", hash = "sha256:d53f8d5d01582ef5960a3c24846de5ea354abf1273f01002825c99f6b050e707", size = 22755, upload-time = "2025-09-15T18:53:51.024Z" } +[[package]] +name = "freezegun" +version = "1.5.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/dd/23e2f4e357f8fd3bdff613c1fe4466d21bfb00a6177f238079b17f7b1c84/freezegun-1.5.5.tar.gz", hash = "sha256:ac7742a6cc6c25a2c35e9292dfd554b897b517d2dec26891a2e8debf205cb94a", size = 35914, upload-time = "2025-08-09T10:39:08.338Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/2e/b41d8a1a917d6581fc27a35d05561037b048e47df50f27f8ac9c7e27a710/freezegun-1.5.5-py3-none-any.whl", hash = "sha256:cd557f4a75cf074e84bc374249b9dd491eaeacd61376b9eb3c423282211619d2", size = 19266, upload-time = "2025-08-09T10:39:06.636Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -111,6 +266,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jmespath" version = "1.0.1" @@ -145,6 +312,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/97/c698bd9350f307daad79dd740806e1a59becd693bd11443a0f531e3229b3/jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6", size = 90379, upload-time = "2022-11-29T20:37:45.842Z" }, ] +[[package]] +name = "librt" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/3f/4ca7dd7819bf8ff303aca39c3c60e5320e46e766ab7f7dd627d3b9c11bdf/librt-0.8.0.tar.gz", hash = "sha256:cb74cdcbc0103fc988e04e5c58b0b31e8e5dd2babb9182b6f9490488eb36324b", size = 177306, upload-time = "2026-02-12T14:53:54.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/53/f3bc0c4921adb0d4a5afa0656f2c0fbe20e18e3e0295e12985b9a5dc3f55/librt-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:17269dd2745dbe8e42475acb28e419ad92dfa38214224b1b01020b8cac70b645", size = 66511, upload-time = "2026-02-12T14:52:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/89/4b/4c96357432007c25a1b5e363045373a6c39481e49f6ba05234bb59a839c1/librt-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f4617cef654fca552f00ce5ffdf4f4b68770f18950e4246ce94629b789b92467", size = 68628, upload-time = "2026-02-12T14:52:31.491Z" }, + { url = "https://files.pythonhosted.org/packages/47/16/52d75374d1012e8fc709216b5eaa25f471370e2a2331b8be00f18670a6c7/librt-0.8.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5cb11061a736a9db45e3c1293cfcb1e3caf205912dfa085734ba750f2197ff9a", size = 198941, upload-time = "2026-02-12T14:52:32.489Z" }, + { url = "https://files.pythonhosted.org/packages/fc/11/d5dd89e5a2228567b1228d8602d896736247424484db086eea6b8010bcba/librt-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4bb00bd71b448f16749909b08a0ff16f58b079e2261c2e1000f2bbb2a4f0a45", size = 210009, upload-time = "2026-02-12T14:52:33.634Z" }, + { url = "https://files.pythonhosted.org/packages/49/d8/fc1a92a77c3020ee08ce2dc48aed4b42ab7c30fb43ce488d388673b0f164/librt-0.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95a719a049f0eefaf1952673223cf00d442952273cbd20cf2ed7ec423a0ef58d", size = 224461, upload-time = "2026-02-12T14:52:34.868Z" }, + { url = "https://files.pythonhosted.org/packages/7f/98/eb923e8b028cece924c246104aa800cf72e02d023a8ad4ca87135b05a2fe/librt-0.8.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bd32add59b58fba3439d48d6f36ac695830388e3da3e92e4fc26d2d02670d19c", size = 217538, upload-time = "2026-02-12T14:52:36.078Z" }, + { url = "https://files.pythonhosted.org/packages/fd/67/24e80ab170674a1d8ee9f9a83081dca4635519dbd0473b8321deecddb5be/librt-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4f764b2424cb04524ff7a486b9c391e93f93dc1bd8305b2136d25e582e99aa2f", size = 225110, upload-time = "2026-02-12T14:52:37.301Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c7/6fbdcbd1a6e5243c7989c21d68ab967c153b391351174b4729e359d9977f/librt-0.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f04ca50e847abc486fa8f4107250566441e693779a5374ba211e96e238f298b9", size = 217758, upload-time = "2026-02-12T14:52:38.89Z" }, + { url = "https://files.pythonhosted.org/packages/4b/bd/4d6b36669db086e3d747434430073e14def032dd58ad97959bf7e2d06c67/librt-0.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9ab3a3475a55b89b87ffd7e6665838e8458e0b596c22e0177e0f961434ec474a", size = 218384, upload-time = "2026-02-12T14:52:40.637Z" }, + { url = "https://files.pythonhosted.org/packages/50/2d/afe966beb0a8f179b132f3e95c8dd90738a23e9ebdba10f89a3f192f9366/librt-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3e36a8da17134ffc29373775d88c04832f9ecfab1880470661813e6c7991ef79", size = 241187, upload-time = "2026-02-12T14:52:43.55Z" }, + { url = "https://files.pythonhosted.org/packages/02/d0/6172ea4af2b538462785ab1a68e52d5c99cfb9866a7caf00fdf388299734/librt-0.8.0-cp312-cp312-win32.whl", hash = "sha256:4eb5e06ebcc668677ed6389164f52f13f71737fc8be471101fa8b4ce77baeb0c", size = 54914, upload-time = "2026-02-12T14:52:44.676Z" }, + { url = "https://files.pythonhosted.org/packages/d4/cb/ceb6ed6175612a4337ad49fb01ef594712b934b4bc88ce8a63554832eb44/librt-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:0a33335eb59921e77c9acc05d0e654e4e32e45b014a4d61517897c11591094f8", size = 62020, upload-time = "2026-02-12T14:52:45.676Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7e/61701acbc67da74ce06ddc7ba9483e81c70f44236b2d00f6a4bfee1aacbf/librt-0.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:24a01c13a2a9bdad20997a4443ebe6e329df063d1978bbe2ebbf637878a46d1e", size = 52443, upload-time = "2026-02-12T14:52:47.218Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + +[[package]] +name = "moto" +version = "5.1.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "cryptography" }, + { name = "jinja2" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "responses" }, + { name = "werkzeug" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/f8/81e2ee90f47a6ae1e475a961bd6a1a1569b04999ba941897b87101b0d5af/moto-5.1.21.tar.gz", hash = "sha256:713dde46e71e2714fa9a29eec513ec618d35e1d84c256331b5aab3f30692feeb", size = 8441171, upload-time = "2026-02-08T21:52:39.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/c7/4b0bc06f0811caa67f7e8c3ca2e637bd8cb4317c2f8839b7d643d7ace68c/moto-5.1.21-py3-none-any.whl", hash = "sha256:311a30095b08b39dd2707f161f1440d361684fe0090b9fd0751dfd1c9b022445", size = 6514163, upload-time = "2026-02-08T21:52:36.91Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -154,6 +411,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -172,6 +438,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -210,48 +485,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, - { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, - { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, - { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, - { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, - { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, - { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, - { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, - { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, - { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, - { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, - { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, - { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, - { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, - { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, - { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, - { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, - { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, - { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, - { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, - { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, - { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, - { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, - { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, - { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, - { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, - { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, - { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, - { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, - { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, - { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, - { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, - { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, - { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, @@ -298,6 +531,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -310,6 +569,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "responses" +version = "0.25.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/95/89c054ad70bfef6da605338b009b2e283485835351a9935c7bfbfaca7ffc/responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4", size = 79320, upload-time = "2025-08-08T19:01:46.709Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/4c/cc276ce57e572c102d9542d383b2cfd551276581dc60004cb94fe8774c11/responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c", size = 34769, upload-time = "2025-08-08T19:01:45.018Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/06/f71e3a86b2df0dfa2d2f72195941cd09b44f87711cb7fa5193732cb9a5fc/ruff-0.14.14.tar.gz", hash = "sha256:2d0f819c9a90205f3a867dbbd0be083bee9912e170fd7d9704cc8ae45824896b", size = 4515732, upload-time = "2026-01-22T22:30:17.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/89/20a12e97bc6b9f9f68343952da08a8099c57237aef953a56b82711d55edd/ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed", size = 10467650, upload-time = "2026-01-22T22:30:08.578Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b1/c5de3fd2d5a831fcae21beda5e3589c0ba67eec8202e992388e4b17a6040/ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c", size = 10883245, upload-time = "2026-01-22T22:30:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7c/3c1db59a10e7490f8f6f8559d1db8636cbb13dccebf18686f4e3c9d7c772/ruff-0.14.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:026c1d25996818f0bf498636686199d9bd0d9d6341c9c2c3b62e2a0198b758de", size = 10231273, upload-time = "2026-01-22T22:30:34.642Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6e/5e0e0d9674be0f8581d1f5e0f0a04761203affce3232c1a1189d0e3b4dad/ruff-0.14.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e", size = 10585753, upload-time = "2026-01-22T22:30:31.781Z" }, + { url = "https://files.pythonhosted.org/packages/23/09/754ab09f46ff1884d422dc26d59ba18b4e5d355be147721bb2518aa2a014/ruff-0.14.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c0f18b922c6d2ff9a5e6c3ee16259adc513ca775bcf82c67ebab7cbd9da5bc8", size = 10286052, upload-time = "2026-01-22T22:30:24.827Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cc/e71f88dd2a12afb5f50733851729d6b571a7c3a35bfdb16c3035132675a0/ruff-0.14.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1629e67489c2dea43e8658c3dba659edbfd87361624b4040d1df04c9740ae906", size = 11043637, upload-time = "2026-01-22T22:30:13.239Z" }, + { url = "https://files.pythonhosted.org/packages/67/b2/397245026352494497dac935d7f00f1468c03a23a0c5db6ad8fc49ca3fb2/ruff-0.14.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:27493a2131ea0f899057d49d303e4292b2cae2bb57253c1ed1f256fbcd1da480", size = 12194761, upload-time = "2026-01-22T22:30:22.542Z" }, + { url = "https://files.pythonhosted.org/packages/5b/06/06ef271459f778323112c51b7587ce85230785cd64e91772034ddb88f200/ruff-0.14.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01ff589aab3f5b539e35db38425da31a57521efd1e4ad1ae08fc34dbe30bd7df", size = 12005701, upload-time = "2026-01-22T22:30:20.499Z" }, + { url = "https://files.pythonhosted.org/packages/41/d6/99364514541cf811ccc5ac44362f88df66373e9fec1b9d1c4cc830593fe7/ruff-0.14.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc12d74eef0f29f51775f5b755913eb523546b88e2d733e1d701fe65144e89b", size = 11282455, upload-time = "2026-01-22T22:29:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/ca/71/37daa46f89475f8582b7762ecd2722492df26421714a33e72ccc9a84d7a5/ruff-0.14.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb8481604b7a9e75eff53772496201690ce2687067e038b3cc31aaf16aa0b974", size = 11215882, upload-time = "2026-01-22T22:29:57.032Z" }, + { url = "https://files.pythonhosted.org/packages/2c/10/a31f86169ec91c0705e618443ee74ede0bdd94da0a57b28e72db68b2dbac/ruff-0.14.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:14649acb1cf7b5d2d283ebd2f58d56b75836ed8c6f329664fa91cdea19e76e66", size = 11180549, upload-time = "2026-01-22T22:30:27.175Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1e/c723f20536b5163adf79bdd10c5f093414293cdf567eed9bdb7b83940f3f/ruff-0.14.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13", size = 10543416, upload-time = "2026-01-22T22:30:01.964Z" }, + { url = "https://files.pythonhosted.org/packages/3e/34/8a84cea7e42c2d94ba5bde1d7a4fae164d6318f13f933d92da6d7c2041ff/ruff-0.14.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412", size = 10285491, upload-time = "2026-01-22T22:30:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/55/ef/b7c5ea0be82518906c978e365e56a77f8de7678c8bb6651ccfbdc178c29f/ruff-0.14.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:cc8b22da8d9d6fdd844a68ae937e2a0adf9b16514e9a97cc60355e2d4b219fc3", size = 10733525, upload-time = "2026-01-22T22:30:06.499Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/aaf1dfbcc53a2811f6cc0a1759de24e4b03e02ba8762daabd9b6bd8c59e3/ruff-0.14.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:16bc890fb4cc9781bb05beb5ab4cd51be9e7cb376bf1dd3580512b24eb3fda2b", size = 11315626, upload-time = "2026-01-22T22:30:36.848Z" }, + { url = "https://files.pythonhosted.org/packages/2c/aa/9f89c719c467dfaf8ad799b9bae0df494513fb21d31a6059cb5870e57e74/ruff-0.14.14-py3-none-win32.whl", hash = "sha256:b530c191970b143375b6a68e6f743800b2b786bbcf03a7965b06c4bf04568167", size = 10502442, upload-time = "2026-01-22T22:30:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/87/44/90fa543014c45560cae1fffc63ea059fb3575ee6e1cb654562197e5d16fb/ruff-0.14.14-py3-none-win_amd64.whl", hash = "sha256:3dde1435e6b6fe5b66506c1dff67a421d0b7f6488d466f651c07f4cab3bf20fd", size = 11630486, upload-time = "2026-01-22T22:30:10.852Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, +] + [[package]] name = "s3transfer" version = "0.14.0" @@ -360,3 +692,24 @@ sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6 wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] + +[[package]] +name = "werkzeug" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/70/1469ef1d3542ae7c2c7b72bd5e3a4e6ee69d7978fa8a3af05a38eca5becf/werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67", size = 864754, upload-time = "2026-01-08T17:49:23.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/e4/8d97cca767bcc1be76d16fb76951608305561c6e056811587f36cb1316a8/werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", size = 225025, upload-time = "2026-01-08T17:49:21.859Z" }, +] + +[[package]] +name = "xmltodict" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/aa/917ceeed4dbb80d2f04dbd0c784b7ee7bba8ae5a54837ef0e5e062cd3cfb/xmltodict-1.0.2.tar.gz", hash = "sha256:54306780b7c2175a3967cad1db92f218207e5bc1aba697d887807c0fb68b7649", size = 25725, upload-time = "2025-09-17T21:59:26.459Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/20/69a0e6058bc5ea74892d089d64dfc3a62ba78917ec5e2cfa70f7c92ba3a5/xmltodict-1.0.2-py3-none-any.whl", hash = "sha256:62d0fddb0dcbc9f642745d8bbf4d81fd17d6dfaec5a15b5c1876300aad92af0d", size = 13893, upload-time = "2025-09-17T21:59:24.859Z" }, +] diff --git a/tasks/granule-invalidator/deploy/outputs.tf b/tasks/granule-invalidator/deploy/outputs.tf index 156154d63052..f7bbaa4c1697 100644 --- a/tasks/granule-invalidator/deploy/outputs.tf +++ b/tasks/granule-invalidator/deploy/outputs.tf @@ -1,19 +1,9 @@ -output "lambda_function_arn" { - description = "ARN of the Lambda function" - value = aws_lambda_function.granule_invalidator_task.arn +output "lambda_function" { + description = "The task lambda function" + value = aws_lambda_function.granule_invalidator_task } -output "lambda_function_name" { - description = "Name of the Lambda function" - value = aws_lambda_function.granule_invalidator_task.function_name -} - -output "lambda_function_invoke_arn" { - description = "Invoke ARN of the Lambda function" - value = aws_lambda_function.granule_invalidator_task.invoke_arn -} - -output "lambda_function_last_modified" { - description = "Last modified date of the Lambda function" - value = aws_lambda_function.granule_invalidator_task.last_modified +output "log_group" { + description = "The task lambda function log group" + value = aws_cloudwatch_log_group.granule_invalidator_task } diff --git a/tf-modules/cumulus/outputs.tf b/tf-modules/cumulus/outputs.tf index 1f4db208862a..e62f4fa7672d 100644 --- a/tf-modules/cumulus/outputs.tf +++ b/tf-modules/cumulus/outputs.tf @@ -55,6 +55,10 @@ output "add_unique_granule_id_task" { value = module.ingest.add_unique_granule_id_task } +output "cnm_response_task" { + value = module.ingest.cnm_response_task +} + output "discover_granules_task" { value = module.ingest.discover_granules_task } diff --git a/tf-modules/ingest/cnm-response.tf b/tf-modules/ingest/cnm-response.tf new file mode 100644 index 000000000000..eb25fd678ae6 --- /dev/null +++ b/tf-modules/ingest/cnm-response.tf @@ -0,0 +1,12 @@ +module "cnm_response_task" { + source = "../../tasks/cnm-response/deploy" + + default_log_retention_days = var.default_log_retention_days + prefix = var.prefix + lambda_processing_role_arn = var.lambda_processing_role_arn + lambda_timeout = lookup(var.lambda_timeouts, "CnmResponse", 300) + lambda_memory_size = lookup(var.lambda_memory_sizes, "CnmResponse", 512) + lambda_subnet_ids = var.lambda_subnet_ids + security_group_id = length(var.lambda_subnet_ids) > 0 ? aws_security_group.no_ingress_all_egress[0].id : "" + tags = var.tags +} diff --git a/tf-modules/ingest/outputs.tf b/tf-modules/ingest/outputs.tf index 17dc8f6acf53..f45ca7f388b9 100644 --- a/tf-modules/ingest/outputs.tf +++ b/tf-modules/ingest/outputs.tf @@ -12,6 +12,14 @@ output "add_unique_granule_id_task" { } } +output "cnm_response_task" { + value = { + task_arn = module.cnm_response_task.lambda_function.arn + task_log_group = module.cnm_response_task.log_group.name + last_modified_date = module.cnm_response_task.lambda_function.last_modified + } +} + output "discover_granules_task" { value = { task_arn = aws_lambda_function.discover_granules_task.arn @@ -43,8 +51,9 @@ output "files_to_granules_task" { output "granule_invalidator_task" { value = { - task_arn = module.granule_invalidator_task.lambda_function_arn - last_modified_date = module.granule_invalidator_task.lambda_function_last_modified + task_arn = module.granule_invalidator_task.lambda_function.arn + task_log_group = module.granule_invalidator_task.log_group.name + last_modified_date = module.granule_invalidator_task.lambda_function.last_modified } } From e51ef27f37cf1f807d793341a6fe6a4fbc69bcc5 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Fri, 20 Feb 2026 13:09:23 -0700 Subject: [PATCH 11/26] CUMULUS-4556 -- Update AJV/minification/CI fixes (#4266) * Update AJV versions * Update aws-sdk versions to ^3.993.0 to address CVE findings * fixup * Update rule tests * Fix bad merge * Fixup * Update kinesis consumer tests * Update minimatch to ^10.2.1 * Add GHSA-3ppc-4f35-3m26 to audit-ci.json * Fix invalid task schemas * Update move-granules schema to remove bogus structure * Update sync-granules schemas * Fix send-pan schema * Update unit test bootsrap * Updating sftp client to handle connection hangups on .end * fixup * Update sftp client to explicitly close the sftpReadStream * Add tmissing tests/cover new cases from sftp client fix * Update lzards backup schema * Update scripts to match changes from CUMULUS-4387 * Fix modified sync granules schema to allow for CMA compatibility * Update CHANGELOG * Fixup * Fixup x2 * Add missing commits * Add stream error listener/handler * Add units demonstrating the event catch case * Fix lodash/noop import * Update json-schema-to-typescript ^15.0.4 --- CHANGELOG.md | 27 ++-- audit-ci.json | 3 +- example/lambdas/s3AccessTest/package.json | 2 +- example/lambdas/snsS3Test/package.json | 2 +- example/package.json | 8 +- .../db-provision-user-database/package.json | 2 +- packages/api-client/package.json | 2 +- packages/api/ecs/async-operation/package.json | 2 +- packages/api/lambdas/message-consumer.js | 4 +- packages/api/lib/schema.js | 4 +- packages/api/models/base.js | 4 +- packages/api/package.json | 10 +- packages/api/tests/endpoints/test-rules.js | 6 +- .../tests/lambdas/test-kinesis-consumer.js | 6 +- packages/async-operations/package.json | 4 +- packages/aws-client/package.json | 42 +++--- packages/common/package.json | 6 +- packages/db/package.json | 2 +- packages/es-client/package.json | 2 +- packages/integration-tests/package.json | 2 +- packages/schemas/package.json | 5 +- packages/sftp-client/src/index.ts | 48 +++++-- packages/sftp-client/tests/test-index.js | 119 ++++++++++++++++ scripts/run_ci_api_unit-coverage.sh | 2 +- scripts/run_ci_unit_coverage.sh | 2 +- .../change-granule-collection-s3/package.json | 2 +- tasks/lzards-backup/schemas/output.json | 134 +++++++++--------- tasks/move-granules/schemas/config.json | 8 +- tasks/move-granules/schemas/output.json | 35 ++--- tasks/queue-granules/schemas/input.json | 4 +- tasks/send-pan/schemas/input.json | 50 +++---- tasks/sync-granule/schemas/config.json | 1 + tasks/sync-granule/schemas/output.json | 36 ++--- .../cumulus-test-cleanup/package.json | 6 +- 34 files changed, 376 insertions(+), 216 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0480b49d49e4..2e54de80162a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Notable Changes + ### Breaking Changes - **CUMULUS-4473** @@ -45,17 +47,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Resolved several integration issues with the granule-invalidator lambda. - Updated packaging script for granule-invalidator to use `uv pip install` instead of `uv sync`. - Added `private_api_lambda_arn` output to the archive module and `private_api_lambda_arn` variable to the ingest module. - - **CSD-91** - Change update-granules-cmr-metadata-file-links task's updateGranuleIdentifiers flag from true to configurable environment variable `update_granule_identifiers`. + +### Fixed -### Notable Changes - -- **CUMULUS-4473** - - Updated Granules Bulk Operations API endpoints to: - - Support `granuleInventoryReportName` and `s3GranuleIdInputFile` in the payload. - - Return consistent output formats across endpoints (previously, some endpoints aggregated errors - while others returned per-granule errors) +- **CUMULUS-4566** + - Updated AJV to ^8.18.0 + - Updated task components to resolve malformed/errant task schemas in the following lambdas: + - SyncGranules + - SendPan + - QueueGranules + - MoveGranules + - LzardsBackup + - ChangeGranuleCollectionS3 + - Update aws-sdk versions to ^3.993.0 +- **CUMULUS-4516** + - Updated sftp-client to explicitly tear down stream in sftp-client/syncFromS3 + - Updated sftp-client to warn/log on `No response from server` errors in `end` method ## [v21.3.0] 2026-01-26 @@ -1580,7 +1589,7 @@ degraded execution table operations. ### Fixed - **CUMULUS-3817** - - updated applicable @aws-sdk dependencies to 3.621.0 to remove inherited vulnerability from fast-xml-parser + - updated applicable @aws-sdk dependencies to 3.993.0 to remove inherited vulnerability from fast-xml-parser - **CUMULUS-3320** - Execution database deletions by `cumulus_id` should have greatly improved performance as a table scan will no longer be required for each record diff --git a/audit-ci.json b/audit-ci.json index 939eba33e1e8..861381312ae8 100644 --- a/audit-ci.json +++ b/audit-ci.json @@ -4,6 +4,7 @@ "retry-count": 20, "allowlist": [ "jsonpath-plus", - "semver" + "semver", + "GHSA-3ppc-4f35-3m26" ] } diff --git a/example/lambdas/s3AccessTest/package.json b/example/lambdas/s3AccessTest/package.json index efb47407f232..2828eda98d28 100644 --- a/example/lambdas/s3AccessTest/package.json +++ b/example/lambdas/s3AccessTest/package.json @@ -18,6 +18,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-s3": "^3.621.0" + "@aws-sdk/client-s3": "^3.993.0" } } diff --git a/example/lambdas/snsS3Test/package.json b/example/lambdas/snsS3Test/package.json index f475206d567c..0a8364af91db 100644 --- a/example/lambdas/snsS3Test/package.json +++ b/example/lambdas/snsS3Test/package.json @@ -18,6 +18,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-s3": "^3.621.0" + "@aws-sdk/client-s3": "^3.993.0" } } diff --git a/example/package.json b/example/package.json index 2bd79d845a5a..937c2acec7f2 100644 --- a/example/package.json +++ b/example/package.json @@ -45,10 +45,10 @@ ] }, "dependencies": { - "@aws-sdk/client-ecs": "^3.621.0", - "@aws-sdk/client-lambda": "^3.621.0", - "@aws-sdk/client-sns": "^3.621.0", - "@aws-sdk/client-sts": "^3.621.0", + "@aws-sdk/client-ecs": "^3.993.0", + "@aws-sdk/client-lambda": "^3.993.0", + "@aws-sdk/client-sns": "^3.993.0", + "@aws-sdk/client-sts": "^3.993.0", "@cumulus/api": "21.3.0", "@cumulus/api-client": "21.3.0", "@cumulus/async-operations": "21.3.0", diff --git a/lambdas/db-provision-user-database/package.json b/lambdas/db-provision-user-database/package.json index a2e3ba02bbeb..b618da6e87a1 100644 --- a/lambdas/db-provision-user-database/package.json +++ b/lambdas/db-provision-user-database/package.json @@ -28,7 +28,7 @@ "failFast": true }, "dependencies": { - "@aws-sdk/client-secrets-manager": "^3.621.0", + "@aws-sdk/client-secrets-manager": "^3.993.0", "@cumulus/common": "21.3.0", "@cumulus/db": "21.3.0", "knex": "2.4.1", diff --git a/packages/api-client/package.json b/packages/api-client/package.json index faf4b7fbcfce..b32fc623066f 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -41,7 +41,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-lambda": "^3.621.0", + "@aws-sdk/client-lambda": "^3.993.0", "@cumulus/aws-client": "21.3.0", "@cumulus/logger": "21.3.0", "p-retry": "^2.0.0" diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index c32c675002fa..77271723a743 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -22,7 +22,7 @@ "coverage": "python ../../../../scripts/coverage_handler/coverage.py" }, "dependencies": { - "@aws-sdk/client-lambda": "^3.621.0", + "@aws-sdk/client-lambda": "^3.993.0", "@cumulus/aws-client": "21.3.0", "@cumulus/db": "21.3.0", "@cumulus/logger": "21.3.0", diff --git a/packages/api/lambdas/message-consumer.js b/packages/api/lambdas/message-consumer.js index a56067e9709d..0c8e4b1b3ad3 100644 --- a/packages/api/lambdas/message-consumer.js +++ b/packages/api/lambdas/message-consumer.js @@ -1,6 +1,6 @@ 'use strict'; -const Ajv = require('ajv'); +const Ajv = require('ajv').default; const get = require('lodash/get'); const set = require('lodash/set'); const { sns } = require('@cumulus/aws-client/services'); @@ -28,7 +28,7 @@ const { function validateMessage(event, originalMessageSource, messageSchema) { if (originalMessageSource === 'sns') return Promise.resolve(event); - const ajv = new Ajv({ allErrors: true }); + const ajv = new Ajv({ allErrors: true, strict: false }); const validate = ajv.compile(messageSchema); return validate(event); } diff --git a/packages/api/lib/schema.js b/packages/api/lib/schema.js index aceddb93be07..dc3bb78e0648 100644 --- a/packages/api/lib/schema.js +++ b/packages/api/lib/schema.js @@ -1,6 +1,6 @@ 'use strict'; -const Ajv = require('ajv'); +const Ajv = require('ajv').default; const cloneDeep = require('lodash/cloneDeep'); const recordIsValid = (incomingRecord, schema, removeAdditional = false) => { @@ -30,7 +30,7 @@ const recordIsValid = (incomingRecord, schema, removeAdditional = false) => { const ajv = new Ajv({ removeAdditional, useDefaults: true, - v5: true, + strict: false, }); const validate = ajv.compile(schemaWithAdditionalPropertiesProhibited); const valid = validate(record); diff --git a/packages/api/models/base.js b/packages/api/models/base.js index 195a87dc9c0c..1171219e497a 100644 --- a/packages/api/models/base.js +++ b/packages/api/models/base.js @@ -2,7 +2,7 @@ const get = require('lodash/get'); const isEqual = require('lodash/isEqual'); -const Ajv = require('ajv'); +const Ajv = require('ajv').default; const pWaitFor = require('p-wait-for'); const awsServices = require('@cumulus/aws-client/services'); @@ -110,7 +110,7 @@ class Manager { const ajv = new Ajv({ removeAdditional, useDefaults: true, - v5: true, + strict: false, }); const validate = ajv.compile(schemaWithAdditionalPropertiesProhibited); const valid = validate(item); diff --git a/packages/api/package.json b/packages/api/package.json index 239d14dc3e08..586fdef26dc4 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -51,10 +51,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-lambda": "^3.621.0", - "@aws-sdk/client-sfn": "^3.621.0", - "@aws-sdk/client-sns": "^3.621.0", - "@aws-sdk/client-sqs": "^3.621.0", + "@aws-sdk/client-lambda": "^3.993.0", + "@aws-sdk/client-sfn": "^3.993.0", + "@aws-sdk/client-sns": "^3.993.0", + "@aws-sdk/client-sqs": "^3.993.0", "@cumulus/api-client": "21.3.0", "@cumulus/async-operations": "21.3.0", "@cumulus/aws-client": "21.3.0", @@ -78,7 +78,7 @@ "@hapi/boom": "^10.0.0", "@mapbox/dyno": "^1.4.2", "aggregate-error": "^3.1.0", - "ajv": "^6.12.3", + "ajv": "^8.18.0", "aws-serverless-express": "^3.3.5", "body-parser": "^1.18.3", "commander": "^2.15.0", diff --git a/packages/api/tests/endpoints/test-rules.js b/packages/api/tests/endpoints/test-rules.js index afdec634c3cb..161d946ea971 100644 --- a/packages/api/tests/endpoints/test-rules.js +++ b/packages/api/tests/endpoints/test-rules.js @@ -899,7 +899,7 @@ test('POST returns a 400 response if rule type is invalid', async (t) => { .expect(400); const { message } = response.body; t.is(response.status, 400); - const regexp = new RegExp('The record has validation errors:.*rule.type.*should be equal to one of the allowed values'); + const regexp = new RegExp('The record has validation errors:.*rule/type.*must be equal to one of the allowed values'); t.truthy(message.match(regexp)); }); @@ -1293,7 +1293,7 @@ test('PATCH returns a 400 response if rule type is invalid', async (t) => { .expect(400); const { message } = response.body; - const regexp = new RegExp('The record has validation errors:.*rule.type.*should be equal to one of the allowed values'); + const regexp = new RegExp('The record has validation errors:.*rule/type.*must be equal to one of the allowed values'); t.truthy(message.match(regexp)); }); @@ -1732,7 +1732,7 @@ test('PUT returns a 400 response if rule type is invalid', async (t) => { .expect(400); const { message } = response.body; - const regexp = new RegExp('The record has validation errors:.*rule.type.*should be equal to one of the allowed values'); + const regexp = new RegExp('The record has validation errors:.*rule/type.*must be equal to one of the allowed values'); t.truthy(message.match(regexp)); }); diff --git a/packages/api/tests/lambdas/test-kinesis-consumer.js b/packages/api/tests/lambdas/test-kinesis-consumer.js index 5bfb0b0d6e6e..72424eefdb0b 100644 --- a/packages/api/tests/lambdas/test-kinesis-consumer.js +++ b/packages/api/tests/lambdas/test-kinesis-consumer.js @@ -172,7 +172,7 @@ test.serial('An SNS fallback retry, should throw an error if message does not in { message: 'validation failed' } ); - t.is(error.errors[0].message, 'should have required property \'collection\''); + t.is(error.errors[0].message, 'must have required property \'collection\''); }); test.serial('A kinesis message, should publish the invalid records to fallbackSNS if the message collection has wrong data type', async (t) => { @@ -200,8 +200,8 @@ test.serial('An SNS Fallback retry, should throw an error if message collection { message: 'validation failed' } ); - t.is(error.errors[0].dataPath, '.collection'); - t.is(error.errors[0].message, 'should be string'); + t.is(error.errors[0].instancePath, '/collection'); + t.is(error.errors[0].message, 'must be string'); }); test.serial('A kinesis message, should publish the invalid record to fallbackSNS if message is invalid json', async (t) => { diff --git a/packages/async-operations/package.json b/packages/async-operations/package.json index 46830f5d8aa9..cf081ede8c66 100644 --- a/packages/async-operations/package.json +++ b/packages/async-operations/package.json @@ -30,8 +30,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-ecs": "^3.621.0", - "@aws-sdk/client-lambda": "^3.621.0", + "@aws-sdk/client-ecs": "^3.993.0", + "@aws-sdk/client-lambda": "^3.993.0", "@cumulus/aws-client": "21.3.0", "@cumulus/db": "21.3.0", "@cumulus/errors": "21.3.0", diff --git a/packages/aws-client/package.json b/packages/aws-client/package.json index ea0d7581c1e2..6143604f5f25 100644 --- a/packages/aws-client/package.json +++ b/packages/aws-client/package.json @@ -47,27 +47,27 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-api-gateway": "^3.621.0", - "@aws-sdk/client-cloudformation": "^3.621.0", - "@aws-sdk/client-cloudwatch-events": "^3.621.0", - "@aws-sdk/client-dynamodb": "^3.621.0", - "@aws-sdk/client-dynamodb-streams": "^3.621.0", - "@aws-sdk/client-ec2": "^3.621.0", - "@aws-sdk/client-ecs": "^3.621.0", - "@aws-sdk/client-elasticsearch-service": "^3.621.0", - "@aws-sdk/client-kinesis": "^3.621.0", - "@aws-sdk/client-kms": "^3.621.0", - "@aws-sdk/client-lambda": "^3.621.0", - "@aws-sdk/client-s3": "^3.621.0", - "@aws-sdk/client-secrets-manager": "^3.621.0", - "@aws-sdk/client-sfn": "^3.621.0", - "@aws-sdk/client-sns": "^3.621.0", - "@aws-sdk/client-sqs": "^3.621.0", - "@aws-sdk/client-sts": "^3.621.0", - "@aws-sdk/lib-dynamodb": "^3.621.0", - "@aws-sdk/lib-storage": "^3.621.0", - "@aws-sdk/s3-request-presigner": "^3.621.0", - "@aws-sdk/signature-v4-crt": "^3.621.0", + "@aws-sdk/client-api-gateway": "^3.993.0", + "@aws-sdk/client-cloudformation": "^3.993.0", + "@aws-sdk/client-cloudwatch-events": "^3.993.0", + "@aws-sdk/client-dynamodb": "^3.993.0", + "@aws-sdk/client-dynamodb-streams": "^3.993.0", + "@aws-sdk/client-ec2": "^3.993.0", + "@aws-sdk/client-ecs": "^3.993.0", + "@aws-sdk/client-elasticsearch-service": "^3.993.0", + "@aws-sdk/client-kinesis": "^3.993.0", + "@aws-sdk/client-kms": "^3.993.0", + "@aws-sdk/client-lambda": "^3.993.0", + "@aws-sdk/client-s3": "^3.993.0", + "@aws-sdk/client-secrets-manager": "^3.993.0", + "@aws-sdk/client-sfn": "^3.993.0", + "@aws-sdk/client-sns": "^3.993.0", + "@aws-sdk/client-sqs": "^3.993.0", + "@aws-sdk/client-sts": "^3.993.0", + "@aws-sdk/lib-dynamodb": "^3.993.0", + "@aws-sdk/lib-storage": "^3.993.0", + "@aws-sdk/s3-request-presigner": "^3.993.0", + "@aws-sdk/signature-v4-crt": "^3.993.0", "@aws-sdk/types": "^3.609.0", "@cumulus/checksum": "21.3.0", "@cumulus/errors": "21.3.0", diff --git a/packages/common/package.json b/packages/common/package.json index 27f28921bd7e..c5226dd48b18 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -44,12 +44,12 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-s3": "^3.621.0", - "@aws-sdk/signature-v4-crt": "^3.621.0", + "@aws-sdk/client-s3": "^3.993.0", + "@aws-sdk/signature-v4-crt": "^3.993.0", "@cumulus/aws-client": "21.3.0", "@cumulus/errors": "21.3.0", "@cumulus/logger": "21.3.0", - "ajv": "^6.12.3", + "ajv": "^8.18.0", "follow-redirects": "^1.2.4", "fs-extra": "^5.0.0", "got": "^14.2.1", diff --git a/packages/db/package.json b/packages/db/package.json index dfdab1a60b69..e505222d4a41 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -32,7 +32,7 @@ "node": ">=22.21.1" }, "dependencies": { - "@aws-sdk/client-secrets-manager": "^3.621.0", + "@aws-sdk/client-secrets-manager": "^3.993.0", "@cumulus/aws-client": "21.3.0", "@cumulus/common": "21.3.0", "@cumulus/errors": "21.3.0", diff --git a/packages/es-client/package.json b/packages/es-client/package.json index 287fa2c6a059..9e535d01de18 100644 --- a/packages/es-client/package.json +++ b/packages/es-client/package.json @@ -33,7 +33,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-providers": "^3.621.0", + "@aws-sdk/credential-providers": "^3.993.0", "@cumulus/common": "21.3.0", "@cumulus/errors": "21.3.0", "@cumulus/logger": "21.3.0", diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index e49744e7a6d2..0dbe3fe7ef30 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -30,7 +30,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-lambda": "^3.621.0", + "@aws-sdk/client-lambda": "^3.993.0", "@cumulus/api": "21.3.0", "@cumulus/api-client": "21.3.0", "@cumulus/aws-client": "21.3.0", diff --git a/packages/schemas/package.json b/packages/schemas/package.json index b05f7b0ce85f..fe4672e6399d 100644 --- a/packages/schemas/package.json +++ b/packages/schemas/package.json @@ -43,8 +43,11 @@ }, "author": "Cumulus Authors", "license": "Apache-2.0", + "devDependencies": { + "@types/node": "^22.0.0" + }, "dependencies": { - "json-schema-to-typescript": "^13.0.2", + "json-schema-to-typescript": "^15.0.4", "json-templates": "^4.1.0", "lodash": "^4.17.21" } diff --git a/packages/sftp-client/src/index.ts b/packages/sftp-client/src/index.ts index abe062b3262d..5dcbc6681073 100644 --- a/packages/sftp-client/src/index.ts +++ b/packages/sftp-client/src/index.ts @@ -72,9 +72,17 @@ export class SftpClient { async end(): Promise { if (this.connected) { - await this.sftpClient.end(); - - this.connected = false; + try { + await this.sftpClient.end(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes('No response from server')) { + throw error; + } + log.warn(`SFTP client end ignored: ${message}`); + } finally { + this.connected = false; + } } } @@ -147,18 +155,28 @@ export class SftpClient { const sftpReadStream = await this.sftp.createReadStream(remotePath); - const result = await S3.promiseS3Upload({ - params: { - Bucket: bucket, - Key: key, - Body: sftpReadStream, - ContentType: mime.lookup(key) || undefined, - }, - }); - - log.info(`Finished copying ${remoteUrl} to ${s3uri}`); - - return { s3uri, etag: result.ETag }; + try { + const result = await S3.promiseS3Upload({ + params: { + Bucket: bucket, + Key: key, + Body: sftpReadStream, + ContentType: mime.lookup(key) || undefined, + }, + }); + + log.info(`Finished copying ${remoteUrl} to ${s3uri}`); + + return { s3uri, etag: result.ETag }; + } finally { + // Explicitly destroy the SFTP read stream so the SFTP channel is + // torn down before end() is called. + sftpReadStream.once('error', (err: Error) => { + const message = err instanceof Error ? err.message : String(err); + log.warn(`SFTP read stream cleanup error (ignored): ${message}`); + }); + sftpReadStream.destroy(); + } } logProgress(total_transferred: number, chunk: number, total: number): void { diff --git a/packages/sftp-client/tests/test-index.js b/packages/sftp-client/tests/test-index.js index 63efba1407a6..2c0b784fef48 100644 --- a/packages/sftp-client/tests/test-index.js +++ b/packages/sftp-client/tests/test-index.js @@ -3,6 +3,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); +const noop = require('lodash/noop'); const S3 = require('@cumulus/aws-client/S3'); const test = require('ava'); const { promisify } = require('util'); @@ -80,6 +81,55 @@ test('SftpClient supports ssh keypair authentication', async (t) => { t.true(fileNames.includes('index.html')); }); +test('SftpClient.sftp throws when not connected', (t) => { + const sftpClient = new SftpClient({ + host: '127.0.0.1', + port: '2222', + username: 'user', + password: 'password', + }); + + const error = t.throws(() => sftpClient.sftp); + t.is(error.message, 'Client not connected'); +}); + +test('SftpClient.buildRemoteUrl throws when host is missing', (t) => { + const sftpClient = new SftpClient({ + host: '', + port: '2222', + username: 'user', + password: 'password', + }); + + const error = t.throws(() => sftpClient.buildRemoteUrl('/granules/file.hdf')); + t.is(error.message, 'host is not set'); +}); + +test('SftpClient.end ignores "No response from server" errors', async (t) => { + const sftpClient = new SftpClient({ + host: '127.0.0.1', + port: '2222', + username: 'user', + password: 'password', + }); + + const originalEnd = sftpClient.sftpClient.end; + sftpClient.connected = true; + sftpClient.sftpClient.end = () => { + throw new Error('No response from server'); + }; + + try { + await t.notThrowsAsync(async () => { + await sftpClient.end(); + }); + } finally { + sftpClient.sftpClient.end = originalEnd; + } + + t.false(sftpClient.connected); +}); + test.serial('sftpClient.list() retrieves a list of files', async (t) => { const files = await t.context.sftpClient.list('/'); t.true(files.length > 0); @@ -136,6 +186,75 @@ test.serial('sftpClient.syncToS3() transfers a file from SFTP to S3 with the cor t.is(s3HeadResponse.ContentType, 'application/x-hdf'); }); +test.serial('sftpClient.syncToS3() destroys read stream on upload failure', async (t) => { + const key = `${randomString()}.hdf`; + let destroyed = false; + const fakeStream = { + once: noop, + destroy: () => { + destroyed = true; + }, + }; + + const originalCreateReadStream = t.context.sftpClient.sftp.createReadStream; + const originalPromiseS3Upload = S3.promiseS3Upload; + + t.context.sftpClient.sftp.createReadStream = () => fakeStream; + S3.promiseS3Upload = ({ params }) => { + t.is(params.Body, fakeStream); + throw new Error('upload failed'); + }; + + try { + await t.throwsAsync( + () => t.context.sftpClient.syncToS3( + '/granules/MOD09GQ.A2017224.h27v08.006.2017227165029.hdf', + t.context.s3Bucket, + key + ), + { message: 'upload failed' } + ); + } finally { + t.context.sftpClient.sftp.createReadStream = originalCreateReadStream; + S3.promiseS3Upload = originalPromiseS3Upload; + } + + t.true(destroyed); +}); + +test.serial('sftpClient.syncToS3() swallows error emitted on read stream during cleanup', async (t) => { + const key = `${randomString()}.hdf`; + let errorListener; + const fakeStream = { + once: (event, cb) => { + if (event === 'error') errorListener = cb; + }, + destroy: () => { + // Simulate ssh2 cleanupRequests() firing the error after destroy + if (errorListener) errorListener(new Error('No response from server')); + }, + }; + + const originalCreateReadStream = t.context.sftpClient.sftp.createReadStream; + const originalPromiseS3Upload = S3.promiseS3Upload; + + t.context.sftpClient.sftp.createReadStream = () => fakeStream; + S3.promiseS3Upload = () => Promise.resolve({ ETag: '"abc123"' }); + + try { + await t.notThrowsAsync( + () => t.context.sftpClient.syncToS3( + '/granules/MOD09GQ.A2017224.h27v08.006.2017227165029.hdf', + t.context.s3Bucket, + key + ) + ); + } finally { + t.context.sftpClient.sftp.createReadStream = originalCreateReadStream; + S3.promiseS3Upload = originalPromiseS3Upload; + } +}); + test.serial('sftpClient.syncToS3Fast() transfers a file from SFTP to S3 with the correct content-type', async (t) => { process.env.SFTP_DEBUG = 'true'; const key = `${randomString()}.hdf`; diff --git a/scripts/run_ci_api_unit-coverage.sh b/scripts/run_ci_api_unit-coverage.sh index 7faa96e475bf..70c30a07abf9 100755 --- a/scripts/run_ci_api_unit-coverage.sh +++ b/scripts/run_ci_api_unit-coverage.sh @@ -17,5 +17,5 @@ then grep '✘' ./test_output.txt exit $RESULT fi -npm run coverage:api -- --noRerun +npm run coverage:api -- --no_rerun exit $RESULT diff --git a/scripts/run_ci_unit_coverage.sh b/scripts/run_ci_unit_coverage.sh index e98c4f03d574..5cdd2755ee12 100755 --- a/scripts/run_ci_unit_coverage.sh +++ b/scripts/run_ci_unit_coverage.sh @@ -17,5 +17,5 @@ then grep '✘' ./test_output.txt exit $RESULT fi -npm run coverage:no-api -- --noRerun +npm run coverage:no-api -- --no_rerun exit $RESULT diff --git a/tasks/change-granule-collection-s3/package.json b/tasks/change-granule-collection-s3/package.json index 974d28263f52..0228f41f25dc 100644 --- a/tasks/change-granule-collection-s3/package.json +++ b/tasks/change-granule-collection-s3/package.json @@ -55,7 +55,7 @@ "lodash": "^4.17.21" }, "devDependencies": { - "@aws-sdk/client-sns": "^3.621.0", + "@aws-sdk/client-sns": "^3.993.0", "@cumulus/api": "21.3.0" } } diff --git a/tasks/lzards-backup/schemas/output.json b/tasks/lzards-backup/schemas/output.json index 2651c25c8ff0..1b0ffac86e7c 100644 --- a/tasks/lzards-backup/schemas/output.json +++ b/tasks/lzards-backup/schemas/output.json @@ -49,73 +49,73 @@ "type": "number" } } - }, - "granules": { - "type": "array", - "items": { - "type": "object", - "required": [ - "files", - "granuleId" - ], - "properties": { - "granuleId": { - "type": "string" - }, - "dataType": { - "type": "string" - }, - "version": { - "type": "string" - }, - "collectionId": { - "type": "string" - }, - "producerGranuleId": { - "type": "string" - }, - "files": { - "type": "array", - "items": { - "additionalProperties": false, - "type": "object", - "required": [ - "bucket", - "key" - ], - "properties": { - "bucket": { - "description": "Bucket where file is archived in S3", - "type": "string" - }, - "checksum": { - "description": "Checksum value for file", - "type": "string" - }, - "checksumType": { - "description": "Type of checksum (e.g. md5, sha256, etc)", - "type": "string" - }, - "fileName": { - "description": "Name of file (e.g. file.txt)", - "type": "string" - }, - "key": { - "description": "S3 Key for archived file", - "type": "string" - }, - "size": { - "description": "Size of file (in bytes)", - "type": "number" - }, - "source": { - "description": "Source URI of the file from origin system (e.g. S3, FTP, HTTP)", - "type": "string" - }, - "type": { - "description": "Type of file (e.g. data, metadata, browse)", - "type": "string" - } + } + }, + "granules": { + "type": "array", + "items": { + "type": "object", + "required": [ + "files", + "granuleId" + ], + "properties": { + "granuleId": { + "type": "string" + }, + "dataType": { + "type": "string" + }, + "version": { + "type": "string" + }, + "collectionId": { + "type": "string" + }, + "producerGranuleId": { + "type": "string" + }, + "files": { + "type": "array", + "items": { + "additionalProperties": false, + "type": "object", + "required": [ + "bucket", + "key" + ], + "properties": { + "bucket": { + "description": "Bucket where file is archived in S3", + "type": "string" + }, + "checksum": { + "description": "Checksum value for file", + "type": "string" + }, + "checksumType": { + "description": "Type of checksum (e.g. md5, sha256, etc)", + "type": "string" + }, + "fileName": { + "description": "Name of file (e.g. file.txt)", + "type": "string" + }, + "key": { + "description": "S3 Key for archived file", + "type": "string" + }, + "size": { + "description": "Size of file (in bytes)", + "type": "number" + }, + "source": { + "description": "Source URI of the file from origin system (e.g. S3, FTP, HTTP)", + "type": "string" + }, + "type": { + "description": "Type of file (e.g. data, metadata, browse)", + "type": "string" } } } diff --git a/tasks/move-granules/schemas/config.json b/tasks/move-granules/schemas/config.json index a5f1329a1d2a..e96a32102b50 100644 --- a/tasks/move-granules/schemas/config.json +++ b/tasks/move-granules/schemas/config.json @@ -65,9 +65,11 @@ "title": "Optional Metadata for the Collection", "type": "object", "additionalProperties": true, - "granuleMetadataFileExtension": { - "description": "Set to the file extension for task to look for the granule metadata information. If none is specified, the granule CMR metadata or ISO metadata file is used", - "type": "string" + "properties": { + "granuleMetadataFileExtension": { + "description": "Set to the file extension for task to look for the granule metadata information. If none is specified, the granule CMR metadata or ISO metadata file is used", + "type": "string" + } } } } diff --git a/tasks/move-granules/schemas/output.json b/tasks/move-granules/schemas/output.json index 3e4cddd813a3..1cb859f074dc 100644 --- a/tasks/move-granules/schemas/output.json +++ b/tasks/move-granules/schemas/output.json @@ -6,22 +6,25 @@ "granuleDuplicates": { "type": "object", "additionalProperties": { - "files": { - "type": "array", - "items": { - "type": "object", - "required": [ - "bucket", - "key" - ], - "properties": { - "bucket": { - "description": "Bucket in-process file is being staged in in S3", - "type": "string" - }, - "key": { - "description": "S3 Key for in-process file", - "type": "string" + "type": "object", + "properties": { + "files": { + "type": "array", + "items": { + "type": "object", + "required": [ + "bucket", + "key" + ], + "properties": { + "bucket": { + "description": "Bucket in-process file is being staged in in S3", + "type": "string" + }, + "key": { + "description": "S3 Key for in-process file", + "type": "string" + } } } } diff --git a/tasks/queue-granules/schemas/input.json b/tasks/queue-granules/schemas/input.json index 5148f546f72c..24638c54544d 100644 --- a/tasks/queue-granules/schemas/input.json +++ b/tasks/queue-granules/schemas/input.json @@ -2,10 +2,10 @@ "title": "QueueGranulesInput", "description": "Describes the input and config used by the queue-granules task", "type": "object", - "require": [ "granules" ], + "required": [ "granules" ], "properties": { "pdr": { - "type": "object", + "type": ["object", "null"], "required": ["name", "path"], "properties": { "name": { "type": "string" }, diff --git a/tasks/send-pan/schemas/input.json b/tasks/send-pan/schemas/input.json index 0544dbb70ba6..1dd5c8b991fc 100644 --- a/tasks/send-pan/schemas/input.json +++ b/tasks/send-pan/schemas/input.json @@ -12,31 +12,31 @@ "name": { "type": "string" }, "path": { "type": "string" } } - } - }, - "running": { - "description": "List of execution arns which are queued or running", - "type": "array", - "items": { - "type": "string" - } - }, - "completed": { - "description": "List of completed execution arns", - "type": "array", - "items": { - "type": "string" - } - }, - "failed": { - "description": "List of failed execution arns with reason", - "type": "array", - "items": { - "type": "object", - "required": ["arn", "reason"], - "properties": { - "arn": { "type": "string" }, - "reason": { "type": "string" } + }, + "running": { + "description": "List of execution arns which are queued or running", + "type": "array", + "items": { + "type": "string" + } + }, + "completed": { + "description": "List of completed execution arns", + "type": "array", + "items": { + "type": "string" + } + }, + "failed": { + "description": "List of failed execution arns with reason", + "type": "array", + "items": { + "type": "object", + "required": ["arn", "reason"], + "properties": { + "arn": { "type": "string" }, + "reason": { "type": "string" } + } } } } diff --git a/tasks/sync-granule/schemas/config.json b/tasks/sync-granule/schemas/config.json index f0143a5fbaee..c02422a01435 100644 --- a/tasks/sync-granule/schemas/config.json +++ b/tasks/sync-granule/schemas/config.json @@ -120,6 +120,7 @@ } }, "pdr": { + "type": ["object", "null"], "required": [ "name", "path" diff --git a/tasks/sync-granule/schemas/output.json b/tasks/sync-granule/schemas/output.json index 5dbf2d4086a8..47879d29dc72 100644 --- a/tasks/sync-granule/schemas/output.json +++ b/tasks/sync-granule/schemas/output.json @@ -9,22 +9,25 @@ "granuleDuplicates": { "type": "object", "additionalProperties": { - "files": { - "type": "array", - "items": { - "type": "object", - "required": [ - "bucket", - "key" - ], - "properties": { - "bucket": { - "description": "Bucket in-process file is being staged in in S3", - "type": "string" - }, - "key": { - "description": "S3 Key for in-process file", - "type": "string" + "type": "object", + "properties": { + "files": { + "type": "array", + "items": { + "type": "object", + "required": [ + "bucket", + "key" + ], + "properties": { + "bucket": { + "description": "Bucket in-process file is being staged in in S3", + "type": "string" + }, + "key": { + "description": "S3 Key for in-process file", + "type": "string" + } } } } @@ -112,6 +115,7 @@ } }, "pdr": { + "type": ["object", "null"], "required": [ "name", "path" diff --git a/tf-modules/internal/cumulus-test-cleanup/package.json b/tf-modules/internal/cumulus-test-cleanup/package.json index 9314f86c9c79..c005ede9165c 100644 --- a/tf-modules/internal/cumulus-test-cleanup/package.json +++ b/tf-modules/internal/cumulus-test-cleanup/package.json @@ -15,9 +15,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-kinesis": "^3.621.0", - "@aws-sdk/client-s3": "^3.621.0", - "@aws-sdk/signature-v4-crt": "^3.621.0", + "@aws-sdk/client-kinesis": "^3.993.0", + "@aws-sdk/client-s3": "^3.993.0", + "@aws-sdk/signature-v4-crt": "^3.993.0", "moment": "^2.30.1" }, "private": true From be0f6761949a8f0ec7b51a4b4c11c2b926d0f9e5 Mon Sep 17 00:00:00 2001 From: jennyhliu <34660846+jennyhliu@users.noreply.github.com> Date: Fri, 20 Feb 2026 19:59:00 -0500 Subject: [PATCH 12/26] CUMULUS-4517:Create s3search Module for DuckDB with S3-Backed Tables (#4248) * CUMULUS-451:Create s3search Module for DuckDB with S3-Backed Tables --- CHANGELOG.md | 8 + packages/common/src/util.ts | 22 +- packages/common/tests/test-util.js | 16 +- packages/db/package.json | 1 + packages/db/src/index.ts | 4 + .../db/src/s3search/AsyncOperationS3Search.ts | 33 + .../db/src/s3search/CollectionS3Search.ts | 136 ++ .../db/src/s3search/DuckDBSearchExecutor.ts | 85 ++ packages/db/src/s3search/ExecutionS3Search.ts | 33 + packages/db/src/s3search/GranuleS3Search.ts | 116 ++ packages/db/src/s3search/PdrS3Search.ts | 33 + packages/db/src/s3search/ProviderS3Search.ts | 33 + .../s3search/ReconciliationReportS3Search.ts | 33 + packages/db/src/s3search/RuleS3Search.ts | 33 + packages/db/src/s3search/StatsS3Search.ts | 63 + packages/db/src/s3search/duckdbHelpers.ts | 113 ++ packages/db/src/s3search/s3TableSchemas.ts | 267 ++++ packages/db/src/search/BaseSearch.ts | 126 +- packages/db/src/search/CollectionSearch.ts | 13 +- packages/db/src/search/ExecutionSearch.ts | 6 +- packages/db/src/search/GranuleSearch.ts | 8 +- packages/db/src/search/PdrSearch.ts | 2 +- .../src/search/ReconciliationReportSearch.ts | 2 +- packages/db/src/search/RuleSearch.ts | 8 +- packages/db/src/search/StatsSearch.ts | 40 +- packages/db/src/search/field-mapping.ts | 104 +- packages/db/src/test-duckdb-utils.ts | 158 +++ packages/db/src/test-utils.ts | 6 + packages/db/src/translate/async_operations.ts | 10 +- packages/db/src/translate/collections.ts | 12 +- packages/db/src/translate/executions.ts | 16 +- packages/db/src/translate/granules.ts | 29 +- packages/db/src/translate/pdr.ts | 10 +- packages/db/src/translate/providers.ts | 6 +- .../src/translate/reconciliation_reports.ts | 9 +- packages/db/src/translate/rules.ts | 16 +- packages/db/src/types/file.ts | 2 + .../s3search/test-AsyncOperationS3Search.js | 338 +++++ .../tests/s3search/test-CollectionS3Search.js | 556 ++++++++ .../tests/s3search/test-ExecutionS3Search.js | 699 +++++++++ .../db/tests/s3search/test-GranuleS3Search.js | 1243 +++++++++++++++++ .../db/tests/s3search/test-PdrS3Search.js | 820 +++++++++++ .../tests/s3search/test-ProviderS3Search.js | 350 +++++ .../test-ReconciliationReportS3Search.js | 261 ++++ .../db/tests/s3search/test-RuleS3Search.js | 440 ++++++ .../db/tests/s3search/test-StatsS3Search.js | 488 +++++++ .../db/tests/search/test-ExecutionSearch.js | 4 +- .../db/tests/search/test-GranuleSearch.js | 134 +- .../tests/translate/test-async-operations.js | 12 + .../db/tests/translate/test-collections.js | 17 +- .../db/tests/translate/test-executions.js | 27 +- packages/db/tests/translate/test-granules.js | 50 +- packages/db/tests/translate/test-pdr.js | 21 + packages/db/tests/translate/test-providers.js | 13 +- .../translate/test-reconciliation-reports.js | 21 +- packages/db/tests/translate/test-rules.js | 19 + 56 files changed, 6936 insertions(+), 189 deletions(-) create mode 100644 packages/db/src/s3search/AsyncOperationS3Search.ts create mode 100644 packages/db/src/s3search/CollectionS3Search.ts create mode 100644 packages/db/src/s3search/DuckDBSearchExecutor.ts create mode 100644 packages/db/src/s3search/ExecutionS3Search.ts create mode 100644 packages/db/src/s3search/GranuleS3Search.ts create mode 100644 packages/db/src/s3search/PdrS3Search.ts create mode 100644 packages/db/src/s3search/ProviderS3Search.ts create mode 100644 packages/db/src/s3search/ReconciliationReportS3Search.ts create mode 100644 packages/db/src/s3search/RuleS3Search.ts create mode 100644 packages/db/src/s3search/StatsS3Search.ts create mode 100644 packages/db/src/s3search/duckdbHelpers.ts create mode 100644 packages/db/src/s3search/s3TableSchemas.ts create mode 100644 packages/db/src/test-duckdb-utils.ts create mode 100644 packages/db/tests/s3search/test-AsyncOperationS3Search.js create mode 100644 packages/db/tests/s3search/test-CollectionS3Search.js create mode 100644 packages/db/tests/s3search/test-ExecutionS3Search.js create mode 100644 packages/db/tests/s3search/test-GranuleS3Search.js create mode 100644 packages/db/tests/s3search/test-PdrS3Search.js create mode 100644 packages/db/tests/s3search/test-ProviderS3Search.js create mode 100644 packages/db/tests/s3search/test-ReconciliationReportS3Search.js create mode 100644 packages/db/tests/s3search/test-RuleS3Search.js create mode 100644 packages/db/tests/s3search/test-StatsS3Search.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e54de80162a..a358917e575a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added supporting Terraform for the CnmResponse task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus. - **CUMULUS-4498** - Added `states:StartExecution` action to the `-steprole` IAM role. +- **CUMULUS-4517** + - Added the `@cumulus/db/s3search` module to enable Cumulus record search via S3-backed tables. + The S3Search subclasses inherit from search/BaseSearch, allowing them to reuse existing query + logic while executing search queries on DuckDB and providing custom record translation. + - Updated the `@cumulus/db/search` module to build queries compatible with both PostgreSQL and DuckDB. + - Updated the `@cumulus/db/search` module to support searching on nested JSON fields. + - Updated the `@cumulus/db/translate` `translatePostgres*Record*ToApi*Record*` functions to + correctly handle query results from both PostgreSQL and DuckDB. ### Changed diff --git a/packages/common/src/util.ts b/packages/common/src/util.ts index 9950a1f6a7c1..aa8064d4828f 100644 --- a/packages/common/src/util.ts +++ b/packages/common/src/util.ts @@ -15,6 +15,8 @@ import type { import isPlainObject from 'lodash/isPlainObject'; import curry from 'lodash/curry'; import isNil from 'lodash/isNil'; +import isObject from 'lodash/isObject'; +import isString from 'lodash/isString'; import omitBy from 'lodash/omitBy'; import * as log from './log'; @@ -76,7 +78,7 @@ export const isOneOf = curry( ); export const returnNullOrUndefinedOrDate = ( - dateVal: string | number | null | undefined + dateVal: string | Date | number | null | undefined ) => (isNil(dateVal) ? dateVal : new Date(dateVal)); /** @@ -131,3 +133,21 @@ export const omitDeepBy: OmitDeepBy = (object: any, cb: any) => { return omitByDeepByOnOwnProps(object); }; + +/** + * Safely parses a value if it is a JSON string resulting in an Object or Array. + * Returns the parsed collection or the original input. + * + * @param val + * @returns + */ +export const parseIfJson = (val: unknown): any => { + if (!isString(val)) return val; + + try { + const result = JSON.parse(val); + return isObject(result) ? result : val; + } catch (e) { + return val; + } +}; diff --git a/packages/common/tests/test-util.js b/packages/common/tests/test-util.js index c8bf455f48b0..ea70d71b0ab6 100644 --- a/packages/common/tests/test-util.js +++ b/packages/common/tests/test-util.js @@ -1,6 +1,6 @@ const test = require('ava'); const isNil = require('lodash/isNil'); -const { omitDeepBy } = require('../util'); +const { omitDeepBy, parseIfJson } = require('../util'); test('omitDeepBy returns new object when properties recursively removed based on predictate function', (t) => { const original = { @@ -28,3 +28,17 @@ test('omitDeepBy returns new object when properties recursively removed based on const result = omitDeepBy(original, isNil); t.deepEqual(result, expected); }); + +test('parseIfJson correctly parses strings into objects/arrays or returns the original value', (t) => { + // Parsed Cases (Objects/Arrays) + t.deepEqual(parseIfJson('{"a":1}'), { a: 1 }, 'should parse JSON objects'); + t.deepEqual(parseIfJson('[1, 2]'), [1, 2], 'should parse JSON arrays'); + + // Strict/Bypass Cases (Primitives/Non-strings) + t.is(parseIfJson('123'), '123', 'should return numeric strings as-is'); + t.is(parseIfJson(true), true, 'should return boolean types as-is'); + t.is(parseIfJson(null), null, 'should return null as-is'); + + // Error Case + t.is(parseIfJson('{"bad":'), '{"bad":', 'should return malformed JSON strings as-is'); +}); diff --git a/packages/db/package.json b/packages/db/package.json index e505222d4a41..076ecc19f0a1 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -39,6 +39,7 @@ "@cumulus/logger": "21.3.0", "@cumulus/message": "21.3.0", "@cumulus/types": "21.3.0", + "@duckdb/node-api": "^1.4.4-r.1", "crypto-random-string": "^3.2.0", "is-valid-hostname": "1.0.2", "knex": "2.4.1", diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index da732e6bbaa2..aeab313a0f21 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -175,6 +175,10 @@ export { ReconciliationReportSearch, } from './search/ReconciliationReportSearch'; +export { + CollectionS3Search, +} from './s3search/CollectionS3Search'; + export { AsyncOperationPgModel } from './models/async_operation'; export { BasePgModel } from './models/base'; export { CollectionPgModel } from './models/collection'; diff --git a/packages/db/src/s3search/AsyncOperationS3Search.ts b/packages/db/src/s3search/AsyncOperationS3Search.ts new file mode 100644 index 000000000000..23efbdf089e6 --- /dev/null +++ b/packages/db/src/s3search/AsyncOperationS3Search.ts @@ -0,0 +1,33 @@ +import { DuckDBConnection } from '@duckdb/node-api'; + +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { AsyncOperationSearch } from '../search/AsyncOperationSearch'; +import { QueryEvent } from '../types/search'; + +/** + * Class to build and execute DuckDB search query for asyncOperation + */ +export class AsyncOperationS3Search extends AsyncOperationSearch { + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event); + + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/CollectionS3Search.ts b/packages/db/src/s3search/CollectionS3Search.ts new file mode 100644 index 000000000000..63a4eaabb508 --- /dev/null +++ b/packages/db/src/s3search/CollectionS3Search.ts @@ -0,0 +1,136 @@ +import { Knex } from 'knex'; +import pick from 'lodash/pick'; +import { DuckDBConnection } from '@duckdb/node-api'; + +import Logger from '@cumulus/logger'; + +import { CollectionSearch, Statuses, StatsRecords, CollectionRecordApi } from '../search/CollectionSearch'; +import { TableNames } from '../tables'; +import { translatePostgresCollectionToApiCollection } from '../translate/collections'; +import { PostgresCollectionRecord } from '../types/collection'; +import { QueryEvent } from '../types/search'; +import { prepareBindings } from './duckdbHelpers'; +import { GranuleS3Search } from './GranuleS3Search'; +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; + +const log = new Logger({ sender: '@cumulus/db/CollectionS3Search' }); + +/** + * Class to build and execute DuckDB search query for collections + */ +export class CollectionS3Search extends CollectionSearch { + private dbConnection: DuckDBConnection; + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event); + this.dbConnection = dbConnection; + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Executes stats query to get granules' status aggregation + * + * @param collectionCumulusIds - array of cumulusIds of the collections + * @param knexClient - knex for the stats query + * @returns the collection's granules status' aggregation + */ + protected async retrieveGranuleStats(collectionCumulusIds: number[], knexClient: Knex) + : Promise { + const granulesTable = TableNames.granules; + let statsQuery = knexClient(granulesTable); + + if (this.active) { + const granuleS3Search = new GranuleS3Search( + { queryStringParameters: this.queryStringParameters }, + this.dbConnection + ); + const { countQuery } = granuleS3Search.buildSearchForActiveCollections(knexClient); + statsQuery = countQuery.clear('select'); + } + + statsQuery + .select(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`) + .count('* as count') + .groupBy(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`) + .whereIn(`${granulesTable}.collection_cumulus_id`, collectionCumulusIds); + + log.debug(`retrieveGranuleStats statsQuery: ${statsQuery?.toSQL().sql}`); + const { sql, bindings } = statsQuery.toSQL().toNative(); + const reader = await this.dbConnection.runAndReadAll( + sql, + prepareBindings(bindings) + ); + const results = reader.getRowObjectsJson(); + const reduced = results.reduce((acc: Record, record) => { + const cumulusId = Number(record.collection_cumulus_id); + if (!acc[cumulusId]) { + acc[cumulusId] = { + queued: 0, + completed: 0, + failed: 0, + running: 0, + total: 0, + }; + } + acc[cumulusId][record.status as keyof Statuses] += Number(record.count); + acc[cumulusId]['total'] += Number(record.count); + return acc; + }, {} as StatsRecords); + return reduced; + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres Collection records returned from query + * @param knexClient - knex for the stats query if incldueStats is true + * @returns translated api records + */ + protected async translatePostgresRecordsToApiRecords(pgRecords: PostgresCollectionRecord[], + knexClient: Knex): Promise[]> { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + + const { fields } = this.dbQueryParameters; + let statsRecords: StatsRecords; + const cumulusIds = pgRecords.map((record) => record.cumulus_id); + if (this.includeStats) { + statsRecords = await this.retrieveGranuleStats(cumulusIds, knexClient); + } + + const apiRecords = pgRecords.map((record) => { + const apiRecord: CollectionRecordApi = translatePostgresCollectionToApiCollection( + record + ); + const apiRecordFinal = fields ? pick(apiRecord, fields) : apiRecord; + + if (statsRecords) { + apiRecordFinal.stats = statsRecords[record.cumulus_id] ? statsRecords[record.cumulus_id] + : { + queued: 0, + completed: 0, + failed: 0, + running: 0, + total: 0, + }; + } + return apiRecordFinal; + }); + return apiRecords; + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/DuckDBSearchExecutor.ts b/packages/db/src/s3search/DuckDBSearchExecutor.ts new file mode 100644 index 000000000000..4a24dfc1b425 --- /dev/null +++ b/packages/db/src/s3search/DuckDBSearchExecutor.ts @@ -0,0 +1,85 @@ +import { knex, Knex } from 'knex'; +import { DuckDBConnection } from '@duckdb/node-api'; + +import { prepareBindings } from './duckdbHelpers'; +import { Meta } from '../search/BaseSearch'; +import { DbQueryParameters } from '../types/search'; + +/** + * DuckDBSearchExecutor is a helper class for executing search queries on DuckDB. + * It wraps a DuckDB connection and provides a unified method to: + * - Build queries using Knex (Postgres dialect) + * - Execute them sequentially to avoid prepared statement conflicts + * - Transform raw database records into API-ready records + * + * This class is intended to be used by S3Search subclasses that inherit + * from BaseSearch, allowing them to reuse query logic while providing + * custom record translation. + */ +export class DuckDBSearchExecutor { + private dbConnection: DuckDBConnection; + private knexBuilder: Knex; + private dbQueryParameters: DbQueryParameters; + private getMetaTemplate: () => Meta; + private translateRecords: ( + records: any[], + knexClient: Knex + ) => any[] | Promise; + + constructor(params: { + dbConnection: DuckDBConnection; + dbQueryParameters: DbQueryParameters; + getMetaTemplate: () => Meta; + translateRecords: ( + records: any[], + knexClient: Knex + ) => any[] | Promise; + }) { + this.dbConnection = params.dbConnection; + this.dbQueryParameters = params.dbQueryParameters; + this.getMetaTemplate = params.getMetaTemplate; + this.translateRecords = params.translateRecords; + + // Use pg dialect to generate DuckDB-compatible SQL ($1, $2, etc.) + this.knexBuilder = knex({ client: 'pg' }); + } + + async query( + buildSearch: (knex: Knex) => { + countQuery?: Knex.QueryBuilder; + searchQuery: Knex.QueryBuilder; + } + ) { + const { countQuery, searchQuery } = buildSearch(this.knexBuilder); + const shouldReturnCountOnly = this.dbQueryParameters.countOnly === true; + + const queryConfigs = shouldReturnCountOnly + ? [{ key: 'count', query: countQuery }] + : [ + { key: 'count', query: countQuery }, + { key: 'records', query: searchQuery }, + ]; + + let countResult: any[] = []; + let records: any[] = []; + + // sequential execution (DuckDB cannot handle multiple prepared statements simultaneously) + for (const config of queryConfigs.filter((c) => c.query)) { + const { sql, bindings } = config.query!.clone().toSQL().toNative(); + // eslint-disable-next-line no-await-in-loop + const reader = await this.dbConnection.runAndReadAll(sql, prepareBindings(bindings)); + const result = reader.getRowObjectsJson(); + + if (config.key === 'count') countResult = result; + else records = result; + } + + const meta = this.getMetaTemplate(); + meta.limit = this.dbQueryParameters.limit; + meta.page = this.dbQueryParameters.page; + meta.count = Number(countResult[0]?.count ?? 0); + + const apiRecords = await this.translateRecords(records, this.knexBuilder); + return { meta, results: apiRecords }; + } +} diff --git a/packages/db/src/s3search/ExecutionS3Search.ts b/packages/db/src/s3search/ExecutionS3Search.ts new file mode 100644 index 000000000000..2525c9726133 --- /dev/null +++ b/packages/db/src/s3search/ExecutionS3Search.ts @@ -0,0 +1,33 @@ +import { DuckDBConnection } from '@duckdb/node-api'; + +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { ExecutionSearch } from '../search/ExecutionSearch'; +import { QueryEvent } from '../types/search'; + +/** + * Class to build and execute DuckDB search query for executions + */ +export class ExecutionS3Search extends ExecutionSearch { + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event, false); // disables estimateTableRowCount + + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/GranuleS3Search.ts b/packages/db/src/s3search/GranuleS3Search.ts new file mode 100644 index 000000000000..9d4e94f25aca --- /dev/null +++ b/packages/db/src/s3search/GranuleS3Search.ts @@ -0,0 +1,116 @@ +import { Knex } from 'knex'; +import pick from 'lodash/pick'; +import { DuckDBConnection } from '@duckdb/node-api'; + +import { ApiGranuleRecord } from '@cumulus/types/api/granules'; +import Logger from '@cumulus/logger'; + +import { + getExecutionInfoByGranuleCumulusIds, + getFilesByGranuleCumulusIds, +} from './duckdbHelpers'; +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { GranuleRecord, GranuleSearch } from '../search/GranuleSearch'; +import { QueryEvent } from '../types/search'; +import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; +import { PostgresFileRecord } from '../types/file'; + +const log = new Logger({ sender: '@cumulus/db/GranuleS3Search' }); + +/** + * Class to build and execute DuckDB search query for granules + */ +export class GranuleS3Search extends GranuleSearch { + private dbConnection: DuckDBConnection; + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event, false); // disables estimateTableRowCount + + this.dbConnection = dbConnection; + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @param knexClient - DB client + * @returns translated api records + */ + protected async translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[], knexClient: Knex) + : Promise[]> { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + + const { fields, includeFullRecord } = this.dbQueryParameters; + + const fileMapping: { [key: number]: PostgresFileRecord[] } = {}; + const executionMapping: { [key: number]: { url: string, granule_cumulus_id: number } } = {}; + const cumulusIds = pgRecords.map((record) => record.cumulus_id); + if (includeFullRecord) { + //get files + const files = await getFilesByGranuleCumulusIds({ + connection: this.dbConnection, + granuleCumulusIds: cumulusIds, + knexBuilder: knexClient, + }); + files.forEach((file) => { + if (!(file.granule_cumulus_id in fileMapping)) { + fileMapping[file.granule_cumulus_id] = []; + } + fileMapping[file.granule_cumulus_id].push(file); + }); + + //get Executions + const executions = await getExecutionInfoByGranuleCumulusIds({ + connection: this.dbConnection, + granuleCumulusIds: cumulusIds, + knexBuilder: knexClient, + }); + executions.forEach((execution) => { + if (!(execution.granule_cumulus_id in executionMapping)) { + executionMapping[execution.granule_cumulus_id] = execution; + } + }); + } + const apiRecords = pgRecords.map((item: GranuleRecord) => { + const collectionPgRecord = { + cumulus_id: item.collection_cumulus_id, + name: item.collectionName, + version: item.collectionVersion, + }; + const executionUrls = executionMapping[item.cumulus_id]?.url + ? [{ url: executionMapping[item.cumulus_id].url }] + : []; + const pdr = item.pdrName ? { name: item.pdrName } : undefined; + const providerPgRecord = item.providerName ? { name: item.providerName } : undefined; + const fileRecords = fileMapping[item.cumulus_id] || []; + const apiRecord = translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord: item, + collectionPgRecord, + pdr, + providerPgRecord, + files: fileRecords, + executionUrls, + }); + + return fields ? pick(apiRecord, fields) : apiRecord; + }); + return apiRecords; + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/PdrS3Search.ts b/packages/db/src/s3search/PdrS3Search.ts new file mode 100644 index 000000000000..a39a80b91e29 --- /dev/null +++ b/packages/db/src/s3search/PdrS3Search.ts @@ -0,0 +1,33 @@ +import { DuckDBConnection } from '@duckdb/node-api'; + +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { PdrSearch } from '../search/PdrSearch'; +import { QueryEvent } from '../types/search'; + +/** + * Class to build and execute DuckDB search query for PDRs + */ +export class PdrS3Search extends PdrSearch { + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event); + + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/ProviderS3Search.ts b/packages/db/src/s3search/ProviderS3Search.ts new file mode 100644 index 000000000000..9a91431490b4 --- /dev/null +++ b/packages/db/src/s3search/ProviderS3Search.ts @@ -0,0 +1,33 @@ +import { DuckDBConnection } from '@duckdb/node-api'; + +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { ProviderSearch } from '../search/ProviderSearch'; +import { QueryEvent } from '../types/search'; + +/** + * Class to build and execute DuckDB search query for providers + */ +export class ProviderS3Search extends ProviderSearch { + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event); + + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/ReconciliationReportS3Search.ts b/packages/db/src/s3search/ReconciliationReportS3Search.ts new file mode 100644 index 000000000000..8abe8cc6d733 --- /dev/null +++ b/packages/db/src/s3search/ReconciliationReportS3Search.ts @@ -0,0 +1,33 @@ +import { DuckDBConnection } from '@duckdb/node-api'; + +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { ReconciliationReportSearch } from '../search/ReconciliationReportSearch'; +import { QueryEvent } from '../types/search'; + +/** + * Class to build and execute DuckDB search query for Reconciliation Report + */ +export class ReconciliationReportS3Search extends ReconciliationReportSearch { + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event); + + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/RuleS3Search.ts b/packages/db/src/s3search/RuleS3Search.ts new file mode 100644 index 000000000000..d9acd1737416 --- /dev/null +++ b/packages/db/src/s3search/RuleS3Search.ts @@ -0,0 +1,33 @@ +import { DuckDBConnection } from '@duckdb/node-api'; + +import { DuckDBSearchExecutor } from './DuckDBSearchExecutor'; +import { RuleSearch } from '../search/RuleSearch'; +import { QueryEvent } from '../types/search'; + +/** + * Class to build and execute DuckDB search query for rules + */ +export class RuleS3Search extends RuleSearch { + private duckDBSearchExecutor: DuckDBSearchExecutor; + + constructor(event: QueryEvent, dbConnection: DuckDBConnection) { + super(event); + + this.duckDBSearchExecutor = new DuckDBSearchExecutor({ + dbConnection, + dbQueryParameters: this.dbQueryParameters, + getMetaTemplate: this._metaTemplate.bind(this), + translateRecords: this.translatePostgresRecordsToApiRecords.bind(this), + }); + } + + /** + * Build and execute search query + * + * @returns search result + */ + async query() { + return this.duckDBSearchExecutor.query((knexBuilder) => + this.buildSearch(knexBuilder)); + } +} diff --git a/packages/db/src/s3search/StatsS3Search.ts b/packages/db/src/s3search/StatsS3Search.ts new file mode 100644 index 000000000000..96c2d4604f5e --- /dev/null +++ b/packages/db/src/s3search/StatsS3Search.ts @@ -0,0 +1,63 @@ +import { knex, Knex } from 'knex'; +import { DuckDBConnection } from '@duckdb/node-api'; +import Logger from '@cumulus/logger'; + +import { QueryEvent } from '../types/search'; +import { + ApiAggregateResult, StatsSearch, SummaryResult, TotalSummary, +} from '../search/StatsSearch'; +import { prepareBindings } from './duckdbHelpers'; + +const log = new Logger({ sender: '@cumulus/db/StatsSearch' }); + +/** + * A class to query postgres for the STATS and STATS/AGGREGATE endpoints + */ +class StatsS3Search extends StatsSearch { + private dbConnection: DuckDBConnection; + private knexBuilder: Knex; + + constructor(event: QueryEvent, type: string, dbConnection: DuckDBConnection) { + super(event, type); + this.dbConnection = dbConnection; + // Use 'pg' dialect to generate DuckDB-compatible SQL ($1, $2, etc.) + this.knexBuilder = knex({ client: 'pg' }); + } + + /** + * Queries postgres for a summary of statistics around the granules in the system + * + * @returns the postgres aggregations based on query + */ + public async summary(): Promise { + const aggregateQuery = this.buildSummaryQuery(this.knexBuilder); + log.debug(`summary about to execute query: ${aggregateQuery?.toSQL().sql}`); + + const { sql, bindings } = aggregateQuery.toSQL().toNative(); + const reader = await this.dbConnection.runAndReadAll( + sql, + prepareBindings([...bindings]) // prepareBindings must be imported/defined in scope + ); + + const aggregateQueryRes: TotalSummary[] = reader.getRowObjectsJson() as any[]; + return this.formatSummaryResult(aggregateQueryRes[0]); + } + + /** + * Executes the aggregate search query + * + * @returns the aggregate query results in api format + */ + async aggregate(): Promise { + const { searchQuery } = this.buildSearch(this.knexBuilder); + const { sql, bindings } = searchQuery.toSQL().toNative(); + const reader = await this.dbConnection.runAndReadAll( + sql, + prepareBindings([...bindings]) // prepareBindings must be imported/defined in scope + ); + const records = reader.getRowObjectsJson() as any; + return this.formatAggregateResult(records); + } +} + +export { StatsS3Search }; diff --git a/packages/db/src/s3search/duckdbHelpers.ts b/packages/db/src/s3search/duckdbHelpers.ts new file mode 100644 index 000000000000..e8c90b8c791e --- /dev/null +++ b/packages/db/src/s3search/duckdbHelpers.ts @@ -0,0 +1,113 @@ +import { DuckDBValue, DuckDBConnection } from '@duckdb/node-api'; +import { knex, Knex } from 'knex'; +import Logger from '@cumulus/logger'; +import { TableNames } from '../tables'; +import { PostgresFileRecord } from '../types/file'; + +const log = new Logger({ sender: '@cumulus/db/duckdbHelpers' }); + +export function prepareBindings(bindings: ReadonlyArray): DuckDBValue[] { + return bindings.map((value) => { + if (value instanceof Date) return value.toISOString(); + if (value !== null && typeof value === 'object') return JSON.stringify(value); + return value as DuckDBValue; + }); +} + +/** + * Returns execution records sorted by most recent first for an input + * set of Granule Cumulus IDs. + * + * @param {object} params - The function parameters. + * @param {DuckDBConnection} params.connection - The active DuckDB connection. + * @param {number[]} params.granuleCumulusIds - Array of granule IDs to filter by. + * @param {Knex} [params.knexBuilder] - Optional Knex instance (defaults to 'pg' client). + * @param {number} [params.limit] - Optional limit for the number of records returned. + * @returns {Promise<{ granule_cumulus_id: number, url: string }[]>} + * Array of objects containing granule_cumulus_id and execution url, sorted by timestamp desc. + */ +export const getExecutionInfoByGranuleCumulusIds = async ({ + connection, + granuleCumulusIds, + knexBuilder = knex({ client: 'pg' }), + limit, +}: { + connection: DuckDBConnection, + granuleCumulusIds: number[], + knexBuilder: Knex, + limit?: number +}): Promise<{ granule_cumulus_id: number, url: string }[]> => { + const knexQuery = knexBuilder(TableNames.executions) + .select([ + `${TableNames.executions}.url`, + `${TableNames.granulesExecutions}.granule_cumulus_id`, + ]) + .join( + TableNames.granulesExecutions, + `${TableNames.executions}.cumulus_id`, + `${TableNames.granulesExecutions}.execution_cumulus_id` + ) + .whereIn(`${TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusIds) + .orderBy(`${TableNames.executions}.timestamp`, 'desc'); + + if (limit) knexQuery.limit(limit); + + const { sql, bindings } = knexQuery.toSQL().toNative(); + log.debug(`getExecutionInfoByGranuleCumulusIds query: ${sql}`); + + // Use spread operator to convert ReadonlyArray to mutable array + const reader = await connection.runAndReadAll( + sql, + prepareBindings([...bindings]) + ); + + return reader.getRowObjectsJson() as any[] as { granule_cumulus_id: number, url: string }[]; +}; + +/** + * Searches for file records by granule cumulus IDs using a DuckDB connection. + * + * @param params - Function parameters + * @param params.connection - Active DuckDB connection used to execute the query + * @param params.granuleCumulusIds - Array of granule Cumulus IDs to filter by + * @param [params.columns='*'] - Columns to select (string or string array) + * @param [params.knexBuilder] - Optional Knex instance (defaults to PostgreSQL dialect) + * @returns Promise resolving to an array of normalized `PostgresFileRecord` objects + * @throws If the DuckDB query execution fails + */ +export const getFilesByGranuleCumulusIds = async ({ + connection, + granuleCumulusIds, + columns = '*', + knexBuilder = knex({ client: 'pg' }), +}: { + connection: DuckDBConnection; + granuleCumulusIds: number[]; + columns?: string | string[]; + knexBuilder?: Knex; +}): Promise => { + const knexQuery = knexBuilder(TableNames.files) + .select(columns) + .whereIn('granule_cumulus_id', granuleCumulusIds); + + const { sql, bindings } = knexQuery.toSQL().toNative(); + + // Execute using DuckDB connection + const reader = await connection.runAndReadAll( + sql, + prepareBindings([...bindings]) // prepareBindings must be imported/defined in scope + ); + + const rows = reader.getRowObjectsJson(); + + // Mapping resolves TS errors and handles DuckDB type conversions + return rows.map((row) => ({ + ...row, + // Ensure IDs are numbers (handles DuckDB BigInt/String return types) + cumulus_id: Number(row.cumulus_id), + granule_cumulus_id: Number(row.granule_cumulus_id), + // Convert ISO timestamp strings back to JS Date objects + created_at: row.created_at ? new Date(row.created_at as string) : undefined, + updated_at: row.updated_at ? new Date(row.updated_at as string) : undefined, + })) as any[] as PostgresFileRecord[]; +}; diff --git a/packages/db/src/s3search/s3TableSchemas.ts b/packages/db/src/s3search/s3TableSchemas.ts new file mode 100644 index 000000000000..fd7cfc431dfe --- /dev/null +++ b/packages/db/src/s3search/s3TableSchemas.ts @@ -0,0 +1,267 @@ +export const asyncOperationsS3TableSql = (tableName: string = 'async_operations') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id INTEGER PRIMARY KEY, + id UUID NOT NULL, + description TEXT NOT NULL, + operation_type TEXT NOT NULL, + output JSON, + status TEXT NOT NULL, + task_arn TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT async_operations_id_unique UNIQUE (id), + CONSTRAINT async_operations_status_check + CHECK (status IN ( + 'RUNNING', + 'SUCCEEDED', + 'RUNNER_FAILED', + 'TASK_FAILED' + )), + CONSTRAINT async_operations_operation_type_check + CHECK (operation_type IN ( + 'Bulk Execution Archive', + 'Bulk Execution Delete', + 'Bulk Granules', + 'Bulk Granule Archive', + 'Bulk Granule Delete', + 'Bulk Granule Reingest', + 'Data Migration', + 'Dead-Letter Processing', + 'DLA Migration', + 'ES Index', + 'Kinesis Replay', + 'Migration Count Report', + 'Reconciliation Report', + 'SQS Replay' + )) +);`; + +export const collectionsS3TableSql = (tableName: string = 'collections') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + version TEXT NOT NULL, + sample_file_name TEXT NOT NULL, + granule_id_validation_regex TEXT NOT NULL, + granule_id_extraction_regex TEXT NOT NULL, + files JSON NOT NULL, + process TEXT, + url_path TEXT, + duplicate_handling TEXT, + report_to_ems BOOLEAN, + ignore_files_config_for_discovery BOOLEAN, + meta JSON, + tags JSON, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (name, version), + CHECK (duplicate_handling IN ('error', 'replace', 'skip', 'version')) +);`; + +export const executionsS3TableSql = (tableName: string = 'executions') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id BIGINT PRIMARY KEY, + arn TEXT NOT NULL, + async_operation_cumulus_id INTEGER, + collection_cumulus_id INTEGER, + parent_cumulus_id BIGINT, + cumulus_version TEXT, + url TEXT, + status TEXT NOT NULL, + tasks JSON, + error JSON, + workflow_name TEXT, + duration REAL, + original_payload JSON, + final_payload JSON, + "timestamp" TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + archived BOOLEAN NOT NULL DEFAULT FALSE, + CONSTRAINT executions_arn_unique UNIQUE (arn), + CONSTRAINT executions_url_unique UNIQUE (url), + CONSTRAINT executions_async_operation_cumulus_id_foreign + FOREIGN KEY (async_operation_cumulus_id) + REFERENCES async_operations (cumulus_id), + CONSTRAINT executions_collection_cumulus_id_foreign + FOREIGN KEY (collection_cumulus_id) + REFERENCES collections (cumulus_id), + CONSTRAINT executions_parent_cumulus_id_foreign + FOREIGN KEY (parent_cumulus_id) + REFERENCES ${tableName} (cumulus_id), + CONSTRAINT executions_status_check + CHECK (status IN ('running', 'completed', 'failed', 'unknown')) +);`; + +export const filesS3TableSql = (tableName: string = 'files') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id BIGINT PRIMARY KEY, + granule_cumulus_id BIGINT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + file_size BIGINT, + bucket TEXT NOT NULL, + checksum_type TEXT, + checksum_value TEXT, + file_name TEXT, + key TEXT NOT NULL, + path TEXT, + source TEXT, + type TEXT, + CONSTRAINT files_bucket_key_unique UNIQUE (bucket, key), + CONSTRAINT files_granule_cumulus_id_foreign + FOREIGN KEY (granule_cumulus_id) + REFERENCES granules (cumulus_id) +);`; + +export const granulesS3TableSql = (tableName: string = 'granules') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id BIGINT PRIMARY KEY, + granule_id TEXT NOT NULL, + status TEXT NOT NULL, + collection_cumulus_id INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + published BOOLEAN, + duration REAL, + time_to_archive REAL, + time_to_process REAL, + product_volume BIGINT, + error JSON, + cmr_link TEXT, + pdr_cumulus_id INTEGER, + provider_cumulus_id INTEGER, + beginning_date_time TIMESTAMPTZ, + ending_date_time TIMESTAMPTZ, + last_update_date_time TIMESTAMPTZ, + processing_end_date_time TIMESTAMPTZ, + processing_start_date_time TIMESTAMPTZ, + production_date_time TIMESTAMPTZ, + query_fields JSON, + "timestamp" TIMESTAMPTZ, + producer_granule_id TEXT NOT NULL, + archived BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE (collection_cumulus_id, granule_id), + CHECK (status IN ('running', 'completed', 'failed', 'queued')) +);`; + +export const granulesExecutionsS3TableSql = (tableName: string = 'granules_executions') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + granule_cumulus_id BIGINT NOT NULL, + execution_cumulus_id BIGINT NOT NULL, + CONSTRAINT granules_executions_granule_execution_unique + UNIQUE (granule_cumulus_id, execution_cumulus_id), + CONSTRAINT granules_executions_execution_cumulus_id_foreign + FOREIGN KEY (execution_cumulus_id) + REFERENCES executions (cumulus_id), + CONSTRAINT granules_executions_granule_cumulus_id_foreign + FOREIGN KEY (granule_cumulus_id) + REFERENCES granules (cumulus_id) +);`; + +export const pdrsS3TableSql = (tableName: string = 'pdrs') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id INTEGER PRIMARY KEY, + collection_cumulus_id INTEGER NOT NULL, + provider_cumulus_id INTEGER NOT NULL, + execution_cumulus_id BIGINT, + status TEXT NOT NULL, + name TEXT NOT NULL, + progress REAL, + pan_sent BOOLEAN, + pan_message TEXT, + stats JSON, + address TEXT, + original_url TEXT, + duration REAL, + "timestamp" TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT pdrs_name_unique UNIQUE (name), + CONSTRAINT pdrs_collection_cumulus_id_foreign + FOREIGN KEY (collection_cumulus_id) + REFERENCES collections (cumulus_id), + CONSTRAINT pdrs_execution_cumulus_id_foreign + FOREIGN KEY (execution_cumulus_id) + REFERENCES executions (cumulus_id), + CONSTRAINT pdrs_provider_cumulus_id_foreign + FOREIGN KEY (provider_cumulus_id) + REFERENCES providers (cumulus_id), + CONSTRAINT pdrs_status_check + CHECK (status IN ('running', 'failed', 'completed')) +);`; + +export const providersS3TableSql = (tableName: string = 'providers') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + protocol TEXT NOT NULL DEFAULT 'http', + host TEXT NOT NULL, + port INTEGER, + username TEXT, + password TEXT, + global_connection_limit INTEGER, + private_key TEXT, + cm_key_id TEXT, + certificate_uri TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + allowed_redirects TEXT[], + max_download_time INTEGER, + CONSTRAINT providers_name_unique UNIQUE (name), + CONSTRAINT providers_protocol_check + CHECK (protocol IN ('http', 'https', 'ftp', 'sftp', 's3')) +);`; + +export const reconciliationReportsS3TableSql = (tableName: string = 'reconciliation_reports') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + type TEXT NOT NULL, + status TEXT NOT NULL, + location TEXT, + error JSON, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT reconciliation_reports_name_unique UNIQUE (name), + CONSTRAINT reconciliation_reports_type_check + CHECK (type IN ( + 'Granule Inventory', + 'Granule Not Found', + 'Internal', + 'Inventory', + 'ORCA Backup' + )), + CONSTRAINT reconciliation_reports_status_check + CHECK (status IN ('Generated', 'Pending', 'Failed')) +);`; + +export const rulesS3TableSql = (tableName: string = 'rules') => ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + cumulus_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + workflow TEXT NOT NULL, + collection_cumulus_id INTEGER, + provider_cumulus_id INTEGER, + type TEXT NOT NULL, + enabled BOOLEAN NOT NULL, + value TEXT, + arn TEXT, + log_event_arn TEXT, + execution_name_prefix TEXT, + payload JSON, + meta JSON, + tags JSON, + queue_url TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT rules_name_unique UNIQUE (name), + CONSTRAINT rules_collection_cumulus_id_foreign + FOREIGN KEY (collection_cumulus_id) + REFERENCES collections (cumulus_id), + CONSTRAINT rules_provider_cumulus_id_foreign + FOREIGN KEY (provider_cumulus_id) + REFERENCES providers (cumulus_id), + CONSTRAINT rules_type_check + CHECK (type IN ('onetime', 'scheduled', 'sns', 'kinesis', 'sqs')) +);`; diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index 84a4413f06b1..433f622a8af4 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -11,7 +11,7 @@ import { convertQueryStringToDbQueryParameters } from './queries'; const log = new Logger({ sender: '@cumulus/db/BaseSearch' }); -type Meta = { +export type Meta = { name: string, stack?: string, table?: string, @@ -32,9 +32,33 @@ export const typeToTable: { [key: string]: string } = { }; /** - * Class to build and execute db search query + * BaseSearch + * + * Abstract base class for building and executing database search queries. + * + * Responsibilities: + * - Parse and normalize incoming query string parameters. + * - Build database queries using Knex. + * - Execute queries against PostgreSQL by default. + * - Return standardized search API response format including metadata. + * + * Default Behavior: + * - The `query()` method executes against PostgreSQL using a Knex client. + * + * DuckDB Support: + * - Subclasses that query DuckDB (e.g., *S3Search classes) must override + * the `query()` and related methods + * - DuckDB subclasses are responsible for: + * - Executing queries using a DuckDB connection. + * - Handling sequential execution (to avoid prepared statement conflicts). + * - Translating DuckDB result types (e.g., string dates/JSON) into proper API types. + * + * Design Notes: + * - Query construction logic (e.g., `buildSearch`) is shared across Postgres + * and DuckDB implementations. + * - Execution strategy is delegated to subclasses when a different database + * engine is required. */ - abstract class BaseSearch { readonly type: string; readonly tableName: string; @@ -103,10 +127,32 @@ abstract class BaseSearch { * @returns whether an estimated row count should be returned */ protected shouldEstimateRowcount(countSql: string): boolean { - const isBasicQuery = (countSql === `select count(*) from "${this.tableName}"`); + const isBasicQuery = (countSql === `select count(* as count) from "${this.tableName}"`); return this.dbQueryParameters.estimateTableRowCount === true && isBasicQuery; } + /** + * Build a JSON query expression string for nested fields. + * + * + * @param fullFieldName - Dot-separated JSON path, e.g., 'query_fields.cnm.receivedTime' + * @returns The JSON query path string + * @example + * buildJsonQueryExpression('query_fields.cnm.receivedTime') + * // returns: query_fields -> 'cnm' ->> 'receivedTime' + */ + protected buildJsonQueryExpression = ( + fullFieldName: string + ) => { + const normalizedFieldName = fullFieldName === 'error.Error.keyword' + ? 'error.Error' : fullFieldName; + const [column, ...pathParts] = normalizedFieldName.split('.'); + + return `${column}${pathParts + .map((p, i) => (i === pathParts.length - 1 ? ` ->> '${p}'` : ` -> '${p}'`)) + .join('')}`; + }; + /** * Build the search query * @@ -140,7 +186,7 @@ abstract class BaseSearch { * * @returns metadata template */ - private _metaTemplate(): Meta { + protected _metaTemplate(): Meta { return { name: 'cumulus-api', stack: process.env.stackName, @@ -159,7 +205,7 @@ abstract class BaseSearch { searchQuery: Knex.QueryBuilder, } { const countQuery = knex(this.tableName) - .count('*'); + .count('* as count'); const searchQuery = knex(this.tableName) .select(`${this.tableName}.*`); @@ -202,6 +248,12 @@ abstract class BaseSearch { Object.entries(exists).forEach(([name, value]) => { const queryMethod = value ? 'whereNotNull' : 'whereNull'; const checkNull = value ? 'not null' : 'null'; + if (name.includes('.')) { + [countQuery, searchQuery].forEach((query) => query?.whereRaw( + `(${this.tableName}.${this.buildJsonQueryExpression(name)}) is ${checkNull}` + )); + return; + } switch (name) { case 'collectionName': case 'collectionVersion': @@ -219,10 +271,6 @@ abstract class BaseSearch { case 'asyncOperationId': [countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.async_operation_cumulus_id`)); break; - case 'error': - case 'error.Error': - [countQuery, searchQuery].forEach((query) => query?.whereRaw(`${this.tableName}.error ->> 'Error' is ${checkNull}`)); - break; case 'parentArn': [countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.parent_cumulus_id`)); break; @@ -250,9 +298,26 @@ abstract class BaseSearch { }) { const { countQuery, searchQuery, dbQueryParameters } = params; const { range = {} } = dbQueryParameters ?? this.dbQueryParameters; + const queries = [countQuery, searchQuery]; Object.entries(range).forEach(([name, rangeValues]) => { - const { gte, lte } = rangeValues; + const { gte, lte } = rangeValues ?? {}; + if (!gte && !lte) return; + + if (name.includes('.')) { + const jsonExpr = `(${this.tableName}.${this.buildJsonQueryExpression(name)})`; + if (gte) { + queries.forEach((query) => + query?.whereRaw(`${jsonExpr} >= ?`, [gte])); + } + + if (lte) { + queries.forEach((query) => + query?.whereRaw(`${jsonExpr} <= ?`, [lte])); + } + return; + } + if (gte) { [countQuery, searchQuery].forEach((query) => query?.where(`${this.tableName}.${name}`, '>=', gte)); } @@ -287,6 +352,12 @@ abstract class BaseSearch { const { term = {} } = dbQueryParameters ?? this.dbQueryParameters; Object.entries(term).forEach(([name, value]) => { + if (name.includes('.')) { + [countQuery, searchQuery].forEach((query) => query?.whereRaw( + `(${this.tableName}.${this.buildJsonQueryExpression(name)}) = ?`, value + )); + return; + } switch (name) { case 'collectionName': [countQuery, searchQuery].forEach((query) => query?.where(`${collectionsTable}.name`, value)); @@ -303,10 +374,6 @@ abstract class BaseSearch { case 'pdrName': [countQuery, searchQuery].forEach((query) => query?.where(`${pdrsTable}.name`, value)); break; - case 'error.Error': - [countQuery, searchQuery] - .forEach((query) => value && query?.whereRaw(`${this.tableName}.error->>'Error' = ?`, value)); - break; case 'asyncOperationId': [countQuery, searchQuery].forEach((query) => query?.where(`${asyncOperationsTable}.id`, value)); break; @@ -359,6 +426,12 @@ abstract class BaseSearch { } Object.entries(omit(terms, ['collectionName', 'collectionVersion'])).forEach(([name, value]) => { + if (name.includes('.')) { + [countQuery, searchQuery].forEach((query) => query?.whereRaw( + `(${this.tableName}.${this.buildJsonQueryExpression(name)}) in (${value.map(() => '?').join(',')})`, [...value] + )); + return; + } switch (name) { case 'executionArn': [countQuery, searchQuery].forEach((query) => query?.whereIn(`${executionsTable}.arn`, value)); @@ -369,10 +442,6 @@ abstract class BaseSearch { case 'pdrName': [countQuery, searchQuery].forEach((query) => query?.whereIn(`${pdrsTable}.name`, value)); break; - case 'error.Error': - [countQuery, searchQuery] - .forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' in (${value.map(() => '?').join(',')})`, [...value])); - break; case 'asyncOperationId': [countQuery, searchQuery].forEach((query) => query?.whereIn(`${asyncOperationsTable}.id`, value)); break; @@ -418,6 +487,12 @@ abstract class BaseSearch { })); } Object.entries(omit(term, ['collectionName', 'collectionVersion'])).forEach(([name, value]) => { + if (name.includes('.')) { + [countQuery, searchQuery].forEach((query) => query?.whereRaw( + `(${this.tableName}.${this.buildJsonQueryExpression(name)}) != ?`, value + )); + return; + } switch (name) { case 'executionArn': [countQuery, searchQuery].forEach((query) => query?.whereNot(`${executionsTable}.arn`, value)); @@ -434,9 +509,6 @@ abstract class BaseSearch { case 'parentArn': [countQuery, searchQuery].forEach((query) => query?.whereNot(`${executionsTable}_parent.arn`, value)); break; - case 'error.Error': - [countQuery, searchQuery].forEach((query) => value && query?.whereRaw(`${this.tableName}.error->>'Error' != ?`, value)); - break; default: [countQuery, searchQuery].forEach((query) => query?.whereNot(`${this.tableName}.${name}`, value)); break; @@ -455,19 +527,23 @@ abstract class BaseSearch { searchQuery: Knex.QueryBuilder, dbQueryParameters?: DbQueryParameters, }) { + const customColumns = ['collectionName', 'collectionVersion', 'executionArn', 'providerName', 'pdrName', 'asyncOperationId', 'parentArn']; const { searchQuery, dbQueryParameters } = params; const { sort } = dbQueryParameters || this.dbQueryParameters; sort?.forEach((key) => { - if (key.column.startsWith('error')) { + const prefixedColumn = `${this.tableName}.${key.column}`; + if (key.column.includes('.')) { searchQuery.orderByRaw( - `${this.tableName}.error ->> 'Error' ${key.order}` + `(${this.tableName}.${this.buildJsonQueryExpression(key.column)}) ${key.order}` ); } else if (dbQueryParameters?.collate) { searchQuery.orderByRaw( `${key} collate \"${dbQueryParameters.collate}\"` ); - } else { + } else if (customColumns.includes(key.column)) { searchQuery.orderBy([key]); + } else { + searchQuery.orderBy(prefixedColumn, key.order); } }); } diff --git a/packages/db/src/search/CollectionSearch.ts b/packages/db/src/search/CollectionSearch.ts index a1d891d0aac7..47258ea83c85 100644 --- a/packages/db/src/search/CollectionSearch.ts +++ b/packages/db/src/search/CollectionSearch.ts @@ -14,7 +14,7 @@ import { TableNames } from '../tables'; const log = new Logger({ sender: '@cumulus/db/CollectionSearch' }); -type Statuses = { +export type Statuses = { queued: number, completed: number, failed: number, @@ -22,11 +22,11 @@ type Statuses = { total: number, }; -type StatsRecords = { +export type StatsRecords = { [key: number]: Statuses, }; -interface CollectionRecordApi extends CollectionRecord { +export interface CollectionRecordApi extends CollectionRecord { stats?: Statuses, } @@ -93,8 +93,7 @@ export class CollectionSearch extends BaseSearch { subQuery .clear('select') .select(1) - .where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`)) - .limit(1); + .where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`)); return subQuery; } @@ -129,7 +128,7 @@ export class CollectionSearch extends BaseSearch { * @param knex - knex for the stats query * @returns the collection's granules status' aggregation */ - private async retrieveGranuleStats(collectionCumulusIds: number[], knex: Knex) + protected async retrieveGranuleStats(collectionCumulusIds: number[], knex: Knex) : Promise { const granulesTable = TableNames.granules; let statsQuery = knex(granulesTable); @@ -144,7 +143,7 @@ export class CollectionSearch extends BaseSearch { statsQuery .select(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`) - .count('*') + .count('* as count') .groupBy(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`) .whereIn(`${granulesTable}.collection_cumulus_id`, collectionCumulusIds); diff --git a/packages/db/src/search/ExecutionSearch.ts b/packages/db/src/search/ExecutionSearch.ts index a9e49d4a1183..bafeabac0a06 100644 --- a/packages/db/src/search/ExecutionSearch.ts +++ b/packages/db/src/search/ExecutionSearch.ts @@ -24,9 +24,9 @@ interface ExecutionRecord extends BaseRecord, PostgresExecutionRecord { * Class to build and execute db search query for executions */ export class ExecutionSearch extends BaseSearch { - constructor(event: QueryEvent) { + constructor(event: QueryEvent, enableEstimate = true) { // estimate the table rowcount by default - if (event?.queryStringParameters?.estimateTableRowCount !== 'false') { + if (enableEstimate && event?.queryStringParameters?.estimateTableRowCount !== 'false') { set(event, 'queryStringParameters.estimateTableRowCount', 'true'); } super(event, 'execution'); @@ -86,7 +86,7 @@ export class ExecutionSearch extends BaseSearch { } const countQuery = knex(this.tableName) - .count('*'); + .count('* as count'); if (this.searchCollection()) { countQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`); diff --git a/packages/db/src/search/GranuleSearch.ts b/packages/db/src/search/GranuleSearch.ts index 1ff9a909435e..6455d1eb71ad 100644 --- a/packages/db/src/search/GranuleSearch.ts +++ b/packages/db/src/search/GranuleSearch.ts @@ -17,7 +17,7 @@ import { getExecutionInfoByGranuleCumulusIds } from '../lib/execution'; const log = new Logger({ sender: '@cumulus/db/GranuleSearch' }); -interface GranuleRecord extends BaseRecord, PostgresGranuleRecord { +export interface GranuleRecord extends BaseRecord, PostgresGranuleRecord { collectionName: string, collectionVersion: string, pdrName?: string, @@ -28,9 +28,9 @@ interface GranuleRecord extends BaseRecord, PostgresGranuleRecord { * Class to build and execute db search query for granules */ export class GranuleSearch extends BaseSearch { - constructor(event: QueryEvent) { + constructor(event: QueryEvent, enableEstimate = true) { // estimate the table rowcount by default - if (event?.queryStringParameters?.estimateTableRowCount !== 'false') { + if (enableEstimate && event?.queryStringParameters?.estimateTableRowCount !== 'false') { set(event, 'queryStringParameters.estimateTableRowCount', 'true'); } super(event, 'granule'); @@ -53,7 +53,7 @@ export class GranuleSearch extends BaseSearch { pdrs: pdrsTable, } = TableNames; const countQuery = knex(this.tableName) - .count('*'); + .count('* as count'); const searchQuery = knex(this.tableName) .select(`${this.tableName}.*`) diff --git a/packages/db/src/search/PdrSearch.ts b/packages/db/src/search/PdrSearch.ts index b0f53ae258d4..394a63f6e72d 100644 --- a/packages/db/src/search/PdrSearch.ts +++ b/packages/db/src/search/PdrSearch.ts @@ -45,7 +45,7 @@ export class PdrSearch extends BaseSearch { executions: executionsTable, } = TableNames; const countQuery = knex(this.tableName) - .count('*'); + .count('* as count'); const searchQuery = knex(this.tableName) .select(`${this.tableName}.*`) diff --git a/packages/db/src/search/ReconciliationReportSearch.ts b/packages/db/src/search/ReconciliationReportSearch.ts index a14bb2824262..9418541a12f1 100644 --- a/packages/db/src/search/ReconciliationReportSearch.ts +++ b/packages/db/src/search/ReconciliationReportSearch.ts @@ -34,7 +34,7 @@ export class ReconciliationReportSearch extends BaseSearch { reconciliationReports: reconciliationReportsTable, } = TableNames; const countQuery = knex(this.tableName) - .count('*'); + .count('* as count'); const searchQuery = knex(this.tableName) .select(`${this.tableName}.*`) diff --git a/packages/db/src/search/RuleSearch.ts b/packages/db/src/search/RuleSearch.ts index 6a5ca2709177..24b16413fb7b 100644 --- a/packages/db/src/search/RuleSearch.ts +++ b/packages/db/src/search/RuleSearch.ts @@ -41,7 +41,7 @@ export class RuleSearch extends BaseSearch { } = TableNames; const countQuery = knex(this.tableName) - .count(`${this.tableName}.cumulus_id`); + .count('* as count'); const searchQuery = knex(this.tableName) .select(`${this.tableName}.*`) @@ -103,13 +103,13 @@ export class RuleSearch extends BaseSearch { ): Promise[]> { log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); - const apiRecords = pgRecords.map(async (record) => { + const apiRecords = pgRecords.map((record) => { const providerPgRecord = record.providerName ? { name: record.providerName } : undefined; const collectionPgRecord = record.collectionName ? { name: record.collectionName, version: record.collectionVersion, } : undefined; - const apiRecord = await translatePostgresRuleToApiRuleWithoutDbQuery( + const apiRecord = translatePostgresRuleToApiRuleWithoutDbQuery( record, collectionPgRecord, providerPgRecord @@ -118,6 +118,6 @@ export class RuleSearch extends BaseSearch { ? pick(apiRecord, this.dbQueryParameters.fields) : apiRecord; }); - return await Promise.all(apiRecords); + return apiRecords; } } diff --git a/packages/db/src/search/StatsSearch.ts b/packages/db/src/search/StatsSearch.ts index f904248d4c06..7827eb6328e7 100644 --- a/packages/db/src/search/StatsSearch.ts +++ b/packages/db/src/search/StatsSearch.ts @@ -10,7 +10,7 @@ import { BaseSearch } from './BaseSearch'; const log = new Logger({ sender: '@cumulus/db/StatsSearch' }); -type TotalSummary = { +export type TotalSummary = { count_errors: number, count_collections: number, count_granules: number, @@ -30,7 +30,7 @@ type Summary = { unit: string, }; -type SummaryResult = { +export type SummaryResult = { errors: Summary, granules: Summary, collections: Summary, @@ -48,7 +48,7 @@ type AggregateRes = { count: number, }; -type ApiAggregateResult = { +export type ApiAggregateResult = { meta: Meta, count: AggregateRes[] }; @@ -80,7 +80,7 @@ class StatsSearch extends BaseSearch { * @param result - the postgres query results * @returns the api object with the aggregate statistics */ - private formatAggregateResult(result: Record): ApiAggregateResult { + protected formatAggregateResult(result: Record): ApiAggregateResult { let totalCount = 0; const responses = []; for (const row of Object.keys(result)) { @@ -108,7 +108,7 @@ class StatsSearch extends BaseSearch { * @param result - the knex summary query results * @returns the api object with the summary statistics */ - private formatSummaryResult(result: TotalSummary): SummaryResult { + protected formatSummaryResult(result: TotalSummary): SummaryResult { const timestampTo = this.dbQueryParameters.range?.updated_at?.lte ?? new Date(); const timestampFrom = this.dbQueryParameters.range?.updated_at?.gte ?? new Date(0); const dateto = (timestampTo as Date).toISOString(); @@ -145,15 +145,9 @@ class StatsSearch extends BaseSearch { }; } - /** - * Queries postgres for a summary of statistics around the granules in the system - * - * @param testKnex - the knex client to be used - * @returns the postgres aggregations based on query - */ - public async summary(testKnex?: Knex): Promise { - const knex = testKnex ?? await getKnexClient(); + protected buildSummaryQuery(knex: Knex): Knex.QueryBuilder { const aggregateQuery: Knex.QueryBuilder = knex(this.tableName); + this.buildRangeQuery({ searchQuery: aggregateQuery }); aggregateQuery.select( knex.raw(`COUNT(CASE WHEN ${this.tableName}.error ->> 'Error' is not null THEN 1 END) AS count_errors`), @@ -161,6 +155,18 @@ class StatsSearch extends BaseSearch { knex.raw(`AVG(${this.tableName}.duration) AS avg_processing_time`), knex.raw(`COUNT(DISTINCT ${this.tableName}.collection_cumulus_id) AS count_collections`) ); + return aggregateQuery; + } + + /** + * Queries postgres for a summary of statistics around the granules in the system + * + * @param testKnex - the knex client to be used + * @returns the postgres aggregations based on query + */ + public async summary(testKnex?: Knex): Promise { + const knex = testKnex ?? await getKnexClient(); + const aggregateQuery = this.buildSummaryQuery(knex); log.debug(`summary about to execute query: ${aggregateQuery?.toSQL().sql}`); const aggregateQueryRes: TotalSummary[] = await aggregateQuery; return this.formatSummaryResult(aggregateQueryRes[0]); @@ -197,10 +203,12 @@ class StatsSearch extends BaseSearch { * @param knex - the knex client to be used */ private aggregateQueryField(query: Knex.QueryBuilder, knex: Knex) { - if (this.field?.includes('error.Error')) { - query.select(knex.raw("error ->> 'Error' as aggregatedfield")); + const normalizedKey = this.field === 'error.Error.keyword' ? 'error.Error' : this.field; + if (normalizedKey?.includes('.')) { + const [root, ...nested] = normalizedKey.split('.'); + query.select(knex.raw(`${root} ->> '${nested.join('.')}' as aggregatedfield`)); } else { - query.select(`${this.tableName}.${this.field} as aggregatedfield`); + query.select(`${this.tableName}.${normalizedKey} as aggregatedfield`); } query.modify((queryBuilder) => this.joinTables(queryBuilder)) .count('* as count') diff --git a/packages/db/src/search/field-mapping.ts b/packages/db/src/search/field-mapping.ts index b97645837896..b41fb9dbf13f 100644 --- a/packages/db/src/search/field-mapping.ts +++ b/packages/db/src/search/field-mapping.ts @@ -3,10 +3,14 @@ import Logger from '@cumulus/logger'; const log = new Logger({ sender: '@cumulus/db/field-mapping' }); +type FieldMapper = (value?: string) => Record; + +type TypeMapping = Record; + // functions to map the api search string field name and value to postgres db field -const granuleMapping: { [key: string]: Function } = { - archived: (value?: boolean) => ({ - archived: value, +const granuleMapping: TypeMapping = { + archived: (value?: string) => ({ + archived: value === 'true', }), beginningDateTime: (value?: string) => ({ beginning_date_time: value, @@ -68,13 +72,6 @@ const granuleMapping: { [key: string]: Function } = { error: (value?: string) => ({ error: value, }), - // nested error field - 'error.Error': (value?: string) => ({ - 'error.Error': value, - }), - 'error.Error.keyword': (value?: string) => ({ - 'error.Error': value, - }), // The following fields require querying other tables collectionId: (value?: string) => { const { name, version } = (value && deconstructCollectionId(value)) || {}; @@ -91,7 +88,7 @@ const granuleMapping: { [key: string]: Function } = { }), }; -const asyncOperationMapping : { [key: string]: Function } = { +const asyncOperationMapping : TypeMapping = { createdAt: (value?: string) => ({ created_at: value && new Date(Number(value)), }), @@ -118,7 +115,7 @@ const asyncOperationMapping : { [key: string]: Function } = { }), }; -const collectionMapping : { [key: string]: Function } = { +const collectionMapping : TypeMapping = { createdAt: (value?: string) => ({ created_at: value && new Date(Number(value)), }), @@ -164,7 +161,7 @@ const collectionMapping : { [key: string]: Function } = { }), }; -const executionMapping : { [key: string]: Function } = { +const executionMapping : TypeMapping = { arn: (value?: string) => ({ arn: value, }), @@ -174,13 +171,6 @@ const executionMapping : { [key: string]: Function } = { duration: (value?: string) => ({ duration: value && Number(value), }), - // nested error field - 'error.Error': (value?: string) => ({ - 'error.Error': value, - }), - 'error.Error.keyword': (value?: string) => ({ - 'error.Error': value, - }), execution: (value?: string) => ({ url: value, }), @@ -210,12 +200,12 @@ const executionMapping : { [key: string]: Function } = { collectionVersion: version, }; }, - archived: (value?: boolean) => ({ - archived: value, + archived: (value?: string) => ({ + archived: value === 'true', }), }; -const pdrMapping : { [key: string]: Function } = { +const pdrMapping : TypeMapping = { address: (value?: string) => ({ address: value, }), @@ -265,7 +255,7 @@ const pdrMapping : { [key: string]: Function } = { }), }; -const providerMapping : { [key: string]: Function } = { +const providerMapping : TypeMapping = { allowedRedirects: (value?: string) => ({ allowed_redirects: value?.split(','), }), @@ -313,7 +303,7 @@ const providerMapping : { [key: string]: Function } = { }), }; -const ruleMapping : { [key: string]: Function } = { +const ruleMapping : TypeMapping = { arn: (value?: string) => ({ arn: value, }), @@ -363,7 +353,7 @@ const ruleMapping : { [key: string]: Function } = { }), }; -const reconciliationReportMapping: { [key: string]: Function } = { +const reconciliationReportMapping: TypeMapping = { name: (value?: string) => ({ name: value, }), @@ -390,8 +380,15 @@ const reconciliationReportMapping: { [key: string]: Function } = { }), }; +const nestedRootsByType: Record> = { + execution: new Set(['error']), + granule: new Set(['error', 'queryFields']), + pdr: new Set(['stats']), + reconciliationReport: new Set(['error']), +}; + // type and its mapping -const supportedMappings: { [key: string]: any } = { +const supportedMappings: Record = { granule: granuleMapping, asyncOperation: asyncOperationMapping, collection: collectionMapping, @@ -402,6 +399,27 @@ const supportedMappings: { [key: string]: any } = { reconciliationReport: reconciliationReportMapping, }; +const toSnakeCase = (str: string) => + str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); + +const mapNestedKey = ( + type: string, + key: string +): string | undefined => { + const normalizedKey = key === 'error.Error.keyword' ? 'error.Error' : key; + + const [root, ...nested] = normalizedKey.split('.'); + + const allowedRoots = nestedRootsByType[type]; + if (!allowedRoots || !allowedRoots.has(root)) { + return undefined; + } + + const mappedRoot = toSnakeCase(root); + if (nested.length === 0) return mappedRoot; + return [mappedRoot, ...nested].join('.'); +}; + /** * Map query string field to db field * @@ -413,11 +431,33 @@ const supportedMappings: { [key: string]: any } = { */ export const mapQueryStringFieldToDbField = ( type: string, - queryField: { name: string, value?: string } -): { [key: string]: any } | undefined => { - if (!(supportedMappings[type] && supportedMappings[type][queryField.name])) { - log.warn(`No db mapping field found for type: ${type}, field ${JSON.stringify(queryField)}`); + queryField: { name: string; value?: string } +): Record | undefined => { + const typeMapping = supportedMappings[type]; + + if (!typeMapping) { + log.warn(`No mapping found for type: ${type}`); return undefined; } - return supportedMappings[type] && supportedMappings[type][queryField.name](queryField.value); + + // Exact match (typed + custom logic) + const exactMapper = typeMapping[queryField.name]; + if (exactMapper) { + return exactMapper(queryField.value); + } + + // Nested fallback with type inference + if (queryField.name.includes('.')) { + const mappedKey = mapNestedKey(type, queryField.name); + if (mappedKey) { + return { + [mappedKey]: queryField.value, + }; + } + } + + log.warn( + `No db mapping field found for type: ${type}, field ${JSON.stringify(queryField)}` + ); + return undefined; }; diff --git a/packages/db/src/test-duckdb-utils.ts b/packages/db/src/test-duckdb-utils.ts new file mode 100644 index 000000000000..596d3bb06b0e --- /dev/null +++ b/packages/db/src/test-duckdb-utils.ts @@ -0,0 +1,158 @@ +import type { Knex } from 'knex'; +import { DuckDBInstance, DuckDBConnection } from '@duckdb/node-api'; +import { + asyncOperationsS3TableSql, + collectionsS3TableSql, + executionsS3TableSql, + filesS3TableSql, + granulesS3TableSql, + granulesExecutionsS3TableSql, + providersS3TableSql, + pdrsS3TableSql, + reconciliationReportsS3TableSql, + rulesS3TableSql, +} from './s3search/s3TableSchemas'; +import { prepareBindings } from './s3search/duckdbHelpers'; + +/** + * Creates a DuckDB in-memory instance and sets up S3/httpfs for testing. + * Configures S3-related settings on the DuckDB instance. + * + * @param {string} dbFilePath - The path to the DuckDB database file. Defaults to in-memory + * @returns {Promise<{ instance: DuckDBInstance, connection: DuckDBConnection }>} + * - The created DuckDB instance and the connection object for interacting with the database. + * - The connection is configured with HTTPFS for S3. + */ +export async function setupDuckDBWithS3ForTesting(dbFilePath: string = ':memory:'): Promise<{ + instance: DuckDBInstance; + connection: DuckDBConnection; +}> { + const instance = await DuckDBInstance.create(dbFilePath); + const connection = await instance.connect(); + + // Configure DuckDB HTTPFS for S3 + await connection.run(` + INSTALL httpfs; + LOAD httpfs; + SET s3_region='us-east-1'; + SET s3_access_key_id='test'; + SET s3_secret_access_key='test'; + SET s3_endpoint='localhost:4566'; + SET s3_use_ssl=false; + SET s3_url_style='path'; + `); + + return { instance, connection }; +} + +/** + * Stages data into a temporary DuckDB table, exports it to Parquet (S3), + * and then loads it into the target table. + * + * @template T - Shape of the row object being inserted. + * @param connection - Active DuckDB connection. + * @param knexBuilder - Knex instance used to generate SQL insert statements. + * @param tableName - Name of the destination table. + * @param tableSql - Function that returns the CREATE TABLE SQL for a given table name. + * @param data - A single row or array of rows to insert. + * @param s3Path - Destination S3 path where the staged data will be exported as Parquet. + * @returns Promise that resolves when the staging, export, and load process completes. + */ +export async function stageAndLoadDuckDBTableFromData< + T extends Record +>( + connection: DuckDBConnection, + knexBuilder: Knex, + tableName: string, + tableSql: (tableName: string) => string, + data: T | T[], + s3Path: string +): Promise { + if (!data || (Array.isArray(data) && data.length === 0)) return; + + const rows: T[] = Array.isArray(data) ? data : [data]; + const tmpTableName = `${tableName}_tmp`; + + // Create temporary staging table + await connection.run(tableSql(tmpTableName)); + + // Insert into staging table + if (tableName === 'executions') { + // Execution rows require parent → child ordering + type ExecutionRow = { parent_cumulus_id?: number | null; [key: string]: any }; + const execRows = rows as ExecutionRow[]; + + const parentRows = execRows.filter((r) => !r.parent_cumulus_id); + const childRows = execRows.filter((r) => r.parent_cumulus_id); + + if (parentRows.length > 0) { + const parentInsert = knexBuilder(tmpTableName) + .insert(parentRows) + .toSQL() + .toNative(); + await connection.run(parentInsert.sql, prepareBindings(parentInsert.bindings)); + } + + if (childRows.length > 0) { + const childInsert = knexBuilder(tmpTableName) + .insert(childRows) + .toSQL() + .toNative(); + await connection.run(childInsert.sql, prepareBindings(childInsert.bindings)); + } + } else { + // Generic insert for other tables + const insertQuery = knexBuilder(tmpTableName) + .insert(rows) + .toSQL() + .toNative(); + await connection.run(insertQuery.sql, prepareBindings(insertQuery.bindings)); + } + + // Export staging table to Parquet (S3) + await connection.run(` + COPY ${tmpTableName} + TO '${s3Path}' + (FORMAT PARQUET); + `); + + // Load from staging table into final table + if (tableName === 'executions') { + // Insert parents first + await connection.run(` + INSERT INTO ${tableName} + SELECT * FROM ${tmpTableName} + WHERE parent_cumulus_id IS NULL; + `); + + // Insert children next + await connection.run(` + INSERT INTO ${tableName} + SELECT * FROM ${tmpTableName} + WHERE parent_cumulus_id IS NOT NULL; + `); + } else { + await connection.run(` + INSERT INTO ${tableName} + SELECT * FROM ${tmpTableName}; + `); + } + + // Drop staging table + await connection.run(`DROP TABLE IF EXISTS ${tmpTableName};`); +} + +export async function createDuckDBTables( + connection: DuckDBConnection +): Promise { + await connection.run(asyncOperationsS3TableSql()); + await connection.run(collectionsS3TableSql()); + await connection.run(providersS3TableSql()); + await connection.run(granulesS3TableSql()); + await connection.run(filesS3TableSql()); + await connection.run(executionsS3TableSql()); + await connection.run(granulesExecutionsS3TableSql()); + await connection.run(pdrsS3TableSql()); + await connection.run(reconciliationReportsS3TableSql()); + await connection.run(rulesS3TableSql()); +} diff --git a/packages/db/src/test-utils.ts b/packages/db/src/test-utils.ts index 4d043845343f..f2f3dd14b839 100644 --- a/packages/db/src/test-utils.ts +++ b/packages/db/src/test-utils.ts @@ -124,6 +124,7 @@ export const fakeGranuleRecordFactory = ( producer_granule_id: cryptoRandomString({ length: 5 }), status: 'completed', created_at: new Date(), + updated_at: new Date(), ...params, }); @@ -132,6 +133,8 @@ export const fakeFileRecordFactory = ( ): Omit => ({ bucket: cryptoRandomString({ length: 3 }), key: cryptoRandomString({ length: 3 }), + created_at: new Date(), + updated_at: new Date(), ...params, }); @@ -144,6 +147,8 @@ export const fakeAsyncOperationRecordFactory = ( status: 'RUNNING', output: { test: 'output' }, task_arn: cryptoRandomString({ length: 3 }), + created_at: new Date(), + updated_at: new Date(), ...params, }); @@ -153,6 +158,7 @@ export const fakePdrRecordFactory = ( name: `pdr${cryptoRandomString({ length: 10 })}`, status: 'running', created_at: new Date(), + updated_at: new Date(), ...params, }); diff --git a/packages/db/src/translate/async_operations.ts b/packages/db/src/translate/async_operations.ts index f454fb14497b..e173f6569926 100644 --- a/packages/db/src/translate/async_operations.ts +++ b/packages/db/src/translate/async_operations.ts @@ -1,5 +1,7 @@ +import isObject from 'lodash/isObject'; import omit from 'lodash/omit'; import { toSnake } from 'snake-camel'; +import { returnNullOrUndefinedOrDate } from '@cumulus/common/util'; import { ApiAsyncOperation } from '@cumulus/types/api/async_operations'; import Logger from '@cumulus/logger'; import { PostgresAsyncOperation } from '../types/async_operation'; @@ -68,10 +70,12 @@ export const translatePostgresAsyncOperationToApiAsyncOperation = ( description: pgAsyncOperation.description, operationType: pgAsyncOperation.operation_type, status: pgAsyncOperation.status, - output: pgAsyncOperation.output ? JSON.stringify(pgAsyncOperation.output) : undefined, + output: isObject(pgAsyncOperation.output) + ? JSON.stringify(pgAsyncOperation.output) + : pgAsyncOperation.output, taskArn: pgAsyncOperation.task_arn, - createdAt: pgAsyncOperation.created_at ? pgAsyncOperation.created_at.getTime() : undefined, - updatedAt: pgAsyncOperation.updated_at ? pgAsyncOperation.updated_at.getTime() : undefined, + createdAt: returnNullOrUndefinedOrDate(pgAsyncOperation.created_at)?.getTime(), + updatedAt: returnNullOrUndefinedOrDate(pgAsyncOperation.created_at)?.getTime(), }; return apiAsyncOperation; }; diff --git a/packages/db/src/translate/collections.ts b/packages/db/src/translate/collections.ts index 8c0ffeea1e71..2d8acbd1dfee 100644 --- a/packages/db/src/translate/collections.ts +++ b/packages/db/src/translate/collections.ts @@ -1,6 +1,6 @@ import { NewCollectionRecord, CollectionRecord } from '@cumulus/types/api/collections'; import { PostgresCollection, PostgresCollectionRecord } from '../types/collection'; -const { removeNilProperties } = require('@cumulus/common/util'); +const { parseIfJson, returnNullOrUndefinedOrDate, removeNilProperties } = require('@cumulus/common/util'); /** * Translates a PostgresCollectionRecord object to a `CollectionRecord` API collection object @@ -10,8 +10,8 @@ const { removeNilProperties } = require('@cumulus/common/util'); export const translatePostgresCollectionToApiCollection = ( collectionRecord: PostgresCollectionRecord ): CollectionRecord => removeNilProperties(({ - createdAt: collectionRecord.created_at.getTime(), - updatedAt: collectionRecord.updated_at.getTime(), + createdAt: returnNullOrUndefinedOrDate(collectionRecord.created_at).getTime(), + updatedAt: returnNullOrUndefinedOrDate(collectionRecord.updated_at).getTime(), name: collectionRecord.name, version: collectionRecord.version, process: collectionRecord.process, @@ -19,12 +19,12 @@ export const translatePostgresCollectionToApiCollection = ( duplicateHandling: collectionRecord.duplicate_handling, granuleId: collectionRecord.granule_id_validation_regex, granuleIdExtraction: collectionRecord.granule_id_extraction_regex, - files: collectionRecord.files, + files: parseIfJson(collectionRecord.files), reportToEms: collectionRecord.report_to_ems, sampleFileName: collectionRecord.sample_file_name, ignoreFilesConfigForDiscovery: collectionRecord.ignore_files_config_for_discovery, - meta: collectionRecord.meta, - tags: collectionRecord.tags, + meta: parseIfJson(collectionRecord.meta), + tags: parseIfJson(collectionRecord.tags), })); /** diff --git a/packages/db/src/translate/executions.ts b/packages/db/src/translate/executions.ts index 92867d7f749a..cc1ebe984f77 100644 --- a/packages/db/src/translate/executions.ts +++ b/packages/db/src/translate/executions.ts @@ -6,7 +6,7 @@ import isNull from 'lodash/isNull'; import { RecordDoesNotExist } from '@cumulus/errors'; import { ApiExecution, ApiExecutionRecord } from '@cumulus/types/api/executions'; import Logger from '@cumulus/logger'; -import { removeNilProperties, returnNullOrUndefinedOrDate } from '@cumulus/common/util'; +import { parseIfJson, removeNilProperties, returnNullOrUndefinedOrDate } from '@cumulus/common/util'; import { ValidationError } from '@cumulus/errors'; import { constructCollectionId, deconstructCollectionId } from '@cumulus/message/Collections'; import { PostgresExecution, PostgresExecutionRecord } from '../types/execution'; @@ -35,10 +35,10 @@ export const translatePostgresExecutionToApiExecutionWithoutDbQuery = ({ status: executionRecord.status, arn: executionRecord.arn, duration: executionRecord.duration, - error: executionRecord.error, - tasks: executionRecord.tasks, - originalPayload: executionRecord.original_payload, - finalPayload: executionRecord.final_payload, + error: parseIfJson(executionRecord.error), + tasks: parseIfJson(executionRecord.tasks), + originalPayload: parseIfJson(executionRecord.original_payload), + finalPayload: parseIfJson(executionRecord.final_payload), type: executionRecord.workflow_name, execution: executionRecord.url, cumulusVersion: executionRecord.cumulus_version, @@ -46,9 +46,9 @@ export const translatePostgresExecutionToApiExecutionWithoutDbQuery = ({ collectionId, parentArn, archived: executionRecord.archived, - createdAt: executionRecord.created_at.getTime(), - updatedAt: executionRecord.updated_at.getTime(), - timestamp: executionRecord.timestamp?.getTime(), + createdAt: returnNullOrUndefinedOrDate(executionRecord.created_at)?.getTime(), + updatedAt: returnNullOrUndefinedOrDate(executionRecord.updated_at)?.getTime(), + timestamp: returnNullOrUndefinedOrDate(executionRecord.timestamp)?.getTime(), }; return removeNilProperties(translatedRecord); }; diff --git a/packages/db/src/translate/granules.ts b/packages/db/src/translate/granules.ts index ca802d9a1835..c0cdb532b7c9 100644 --- a/packages/db/src/translate/granules.ts +++ b/packages/db/src/translate/granules.ts @@ -2,7 +2,7 @@ import { Knex } from 'knex'; import { deconstructCollectionId, constructCollectionId } from '@cumulus/message/Collections'; import { ApiGranule, ApiGranuleRecord, GranuleStatus } from '@cumulus/types/api/granules'; -import { removeNilProperties, returnNullOrUndefinedOrDate } from '@cumulus/common/util'; +import { parseIfJson, removeNilProperties, returnNullOrUndefinedOrDate } from '@cumulus/common/util'; import { ValidationError } from '@cumulus/errors'; import isNil from 'lodash/isNil'; import isNull from 'lodash/isNull'; @@ -52,31 +52,36 @@ export const translatePostgresGranuleToApiGranuleWithoutDbQuery = ({ providerPgRecord?: Pick, }): ApiGranuleRecord => removeNilProperties({ archived: granulePgRecord.archived, - beginningDateTime: granulePgRecord.beginning_date_time?.toISOString(), + beginningDateTime: returnNullOrUndefinedOrDate(granulePgRecord.beginning_date_time) + ?.toISOString(), cmrLink: granulePgRecord.cmr_link, collectionId: constructCollectionId(collectionPgRecord.name, collectionPgRecord.version), - createdAt: granulePgRecord.created_at?.getTime(), + createdAt: new Date(granulePgRecord.created_at)?.getTime(), duration: granulePgRecord.duration, - endingDateTime: granulePgRecord.ending_date_time?.toISOString(), - error: granulePgRecord.error, + endingDateTime: returnNullOrUndefinedOrDate(granulePgRecord.ending_date_time)?.toISOString(), + error: parseIfJson(granulePgRecord.error), execution: executionUrls[0] ? executionUrls[0].url : undefined, files: files.length > 0 ? files.map((file) => translatePostgresFileToApiFile(file)) : [], granuleId: granulePgRecord.granule_id, - lastUpdateDateTime: granulePgRecord.last_update_date_time?.toISOString(), + lastUpdateDateTime: returnNullOrUndefinedOrDate(granulePgRecord.last_update_date_time) + ?.toISOString(), pdrName: pdr ? pdr.name : undefined, - processingEndDateTime: granulePgRecord.processing_end_date_time?.toISOString(), - processingStartDateTime: granulePgRecord.processing_start_date_time?.toISOString(), + processingEndDateTime: returnNullOrUndefinedOrDate(granulePgRecord.processing_end_date_time) + ?.toISOString(), + processingStartDateTime: returnNullOrUndefinedOrDate(granulePgRecord.processing_start_date_time) + ?.toISOString(), producerGranuleId: granulePgRecord.producer_granule_id, - productionDateTime: granulePgRecord.production_date_time?.toISOString(), + productionDateTime: returnNullOrUndefinedOrDate(granulePgRecord.production_date_time) + ?.toISOString(), productVolume: granulePgRecord.product_volume, provider: providerPgRecord ? providerPgRecord.name : undefined, published: granulePgRecord.published, - queryFields: granulePgRecord.query_fields, + queryFields: parseIfJson(granulePgRecord.query_fields), status: granulePgRecord.status as GranuleStatus, - timestamp: granulePgRecord.timestamp?.getTime(), + timestamp: returnNullOrUndefinedOrDate(granulePgRecord.timestamp)?.getTime(), timeToArchive: granulePgRecord.time_to_archive, timeToPreprocess: granulePgRecord.time_to_process, - updatedAt: granulePgRecord.updated_at?.getTime(), + updatedAt: new Date(granulePgRecord.updated_at)?.getTime(), }); /** diff --git a/packages/db/src/translate/pdr.ts b/packages/db/src/translate/pdr.ts index d2c010825e0f..e3bf4f97a745 100644 --- a/packages/db/src/translate/pdr.ts +++ b/packages/db/src/translate/pdr.ts @@ -1,6 +1,6 @@ import { Knex } from 'knex'; -import { removeNilProperties } from '@cumulus/common/util'; +import { parseIfJson, removeNilProperties, returnNullOrUndefinedOrDate } from '@cumulus/common/util'; import { constructCollectionId, deconstructCollectionId } from '@cumulus/message/Collections'; import { getExecutionUrlFromArn } from '@cumulus/message/Executions'; import { ApiPdr } from '@cumulus/types/api/pdrs'; @@ -85,17 +85,17 @@ export const translatePostgresPdrToApiPdrWithoutDbQuery = ({ provider: providerPgRecord?.name, collectionId: constructCollectionId(collectionPgRecord.name, collectionPgRecord.version), status: pdrPgRecord.status, - createdAt: pdrPgRecord.created_at.getTime(), + createdAt: returnNullOrUndefinedOrDate(pdrPgRecord.created_at)?.getTime(), progress: pdrPgRecord.progress, execution: executionArn ? getExecutionUrlFromArn(executionArn) : undefined, PANSent: pdrPgRecord.pan_sent, PANmessage: pdrPgRecord.pan_message, - stats: pdrPgRecord.stats, + stats: parseIfJson(pdrPgRecord.stats), address: pdrPgRecord.address, originalUrl: pdrPgRecord.original_url, - timestamp: (pdrPgRecord.timestamp ? pdrPgRecord.timestamp.getTime() : undefined), + timestamp: returnNullOrUndefinedOrDate(pdrPgRecord.timestamp)?.getTime(), duration: pdrPgRecord.duration, - updatedAt: pdrPgRecord.updated_at.getTime(), + updatedAt: returnNullOrUndefinedOrDate(pdrPgRecord.updated_at)?.getTime(), }); /** diff --git a/packages/db/src/translate/providers.ts b/packages/db/src/translate/providers.ts index 79dd98eeefe3..636cb173d382 100644 --- a/packages/db/src/translate/providers.ts +++ b/packages/db/src/translate/providers.ts @@ -1,4 +1,4 @@ -import { removeNilProperties } from '@cumulus/common/util'; +import { removeNilProperties, returnNullOrUndefinedOrDate } from '@cumulus/common/util'; import { envUtils } from '@cumulus/common'; import { KMS } from '@cumulus/aws-client'; import { ApiProvider } from '@cumulus/types'; @@ -25,8 +25,8 @@ export const translatePostgresProviderToApiProvider = ( port: record.port, host: record.host, protocol: record.protocol, - createdAt: record.created_at.getTime(), - updatedAt: record.updated_at.getTime(), + createdAt: returnNullOrUndefinedOrDate(record.created_at)?.getTime(), + updatedAt: returnNullOrUndefinedOrDate(record.updated_at)?.getTime(), username: record.username, password: record.password, allowedRedirects: record.allowed_redirects, diff --git a/packages/db/src/translate/reconciliation_reports.ts b/packages/db/src/translate/reconciliation_reports.ts index 64ec486460ec..0268c556c9bc 100644 --- a/packages/db/src/translate/reconciliation_reports.ts +++ b/packages/db/src/translate/reconciliation_reports.ts @@ -1,7 +1,7 @@ import { ApiReconciliationReportRecord } from '@cumulus/types/api/reconciliation_reports'; import { PostgresReconciliationReport, PostgresReconciliationReportRecord } from '../types/reconciliation_report'; -const { removeNilProperties } = require('@cumulus/common/util'); +const { parseIfJson, removeNilProperties, returnNullOrUndefinedOrDate } = require('@cumulus/common/util'); const pick = require('lodash/pick'); /** @@ -31,9 +31,10 @@ export const translatePostgresReconReportToApiReconReport = ( pgReconciliationReport: PostgresReconciliationReportRecord ): ApiReconciliationReportRecord => { const apiReconciliationReport = removeNilProperties({ - ...pick(pgReconciliationReport, ['name', 'type', 'status', 'location', 'error']), - createdAt: pgReconciliationReport.created_at?.getTime(), - updatedAt: pgReconciliationReport.updated_at?.getTime(), + ...pick(pgReconciliationReport, ['name', 'type', 'status', 'location']), + error: parseIfJson(pgReconciliationReport.error), + createdAt: returnNullOrUndefinedOrDate(pgReconciliationReport.created_at)?.getTime(), + updatedAt: returnNullOrUndefinedOrDate(pgReconciliationReport.updated_at)?.getTime(), }); return apiReconciliationReport; }; diff --git a/packages/db/src/translate/rules.ts b/packages/db/src/translate/rules.ts index 39dafbcae89a..c8ccb493f94e 100644 --- a/packages/db/src/translate/rules.ts +++ b/packages/db/src/translate/rules.ts @@ -1,5 +1,5 @@ import { Knex } from 'knex'; -import { removeNilProperties } from '@cumulus/common/util'; +import { parseIfJson, removeNilProperties } from '@cumulus/common/util'; import { RuleRecord, Rule } from '@cumulus/types/api/rules'; import { CollectionPgModel } from '../models/collection'; @@ -8,11 +8,11 @@ import { PostgresRule, PostgresRuleRecord } from '../types/rule'; import { PostgresProviderRecord } from '../types/provider'; import { PostgresCollectionRecord } from '../types/collection'; -export const translatePostgresRuleToApiRuleWithoutDbQuery = async ( +export const translatePostgresRuleToApiRuleWithoutDbQuery = ( pgRule: PostgresRuleRecord, collectionPgRecord?: Pick, providerPgRecord?: Partial -): Promise => { +): RuleRecord => { const apiRule: RuleRecord = { name: pgRule.name, workflow: pgRule.workflow, @@ -28,13 +28,13 @@ export const translatePostgresRuleToApiRuleWithoutDbQuery = async ( value: pgRule.value, }), state: pgRule.enabled ? 'ENABLED' : 'DISABLED', - meta: pgRule.meta, - payload: pgRule.payload, + meta: parseIfJson(pgRule.meta), + payload: parseIfJson(pgRule.payload), executionNamePrefix: pgRule.execution_name_prefix, queueUrl: pgRule.queue_url, - tags: pgRule.tags, - createdAt: pgRule.created_at.getTime(), - updatedAt: pgRule.updated_at.getTime(), + tags: parseIfJson(pgRule.tags), + createdAt: new Date(pgRule.created_at).getTime(), + updatedAt: new Date(pgRule.updated_at).getTime(), }; return removeNilProperties(apiRule); }; diff --git a/packages/db/src/types/file.ts b/packages/db/src/types/file.ts index 916d46dbd1c1..136062554572 100644 --- a/packages/db/src/types/file.ts +++ b/packages/db/src/types/file.ts @@ -4,11 +4,13 @@ export interface PostgresFile { granule_cumulus_id: number, checksum_type?: string, checksum_value?: string, + created_at?: Date | null, file_name?: string, file_size?: number, path?: string, source?: string, type?: string, + updated_at?: Date | null, } // file_size is stored as a BigInt in Postgres. It returns from PG to Node diff --git a/packages/db/tests/s3search/test-AsyncOperationS3Search.js b/packages/db/tests/s3search/test-AsyncOperationS3Search.js new file mode 100644 index 000000000000..4d22b411a3dc --- /dev/null +++ b/packages/db/tests/s3search/test-AsyncOperationS3Search.js @@ -0,0 +1,338 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const { v4: uuidv4 } = require('uuid'); +const omit = require('lodash/omit'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + asyncOperationsS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { AsyncOperationS3Search } = require('../../dist/s3search/AsyncOperationS3Search'); +const { + translatePostgresAsyncOperationToApiAsyncOperation, +} = require('../../dist/translate/async_operations'); +const { + fakeAsyncOperationRecordFactory, +} = require('../../dist'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + t.context.asyncOperationSearchTimestamp = 1579352700000; + t.context.asyncOperations = range(100).map((num) => ( + fakeAsyncOperationRecordFactory({ + cumulus_id: num, + updated_at: new Date(t.context.asyncOperationSearchTimestamp + (num % 2)), + operation_type: num % 2 === 0 ? 'Bulk Granules' : 'Data Migration', + task_arn: num % 2 === 0 ? cryptoRandomString({ length: 3 }) : undefined, + }) + )); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create asyncOperation'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'async_operations', + asyncOperationsS3TableSql, + t.context.asyncOperations, + `${duckdbS3Prefix}async_operations.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('AsyncOperationS3Search returns 10 async operations by default', async (t) => { + const { connection } = t.context; + const dbSearch = new AsyncOperationS3Search({}, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 100); + t.is(results.length, 10); +}); + +test.serial('AsyncOperationS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test.serial('AsyncOperationS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 20, + infix: t.context.asyncOperations[5].id.slice(1), + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('AsyncOperationS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 20, + prefix: t.context.asyncOperations[5].id.slice(0, -1), + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('AsyncOperationS3Search supports term search for uuid field', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.asyncOperations[5]; + const queryStringParameters = { + limit: 200, + id: dbRecord.id, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('AsyncOperationS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: `${t.context.asyncOperationSearchTimestamp + 1}`, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test.serial('AsyncOperationS3Search supports term search for _id field', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.asyncOperations[5]; + const queryStringParameters = { + limit: 200, + _id: dbRecord.id, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('AsyncOperationS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + operationType: 'Bulk Granules', + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test.serial('AsyncOperationS3Search supports range search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + timestamp__from: `${t.context.asyncOperationSearchTimestamp + 1}`, + timestamp__to: `${t.context.asyncOperationSearchTimestamp + 2}`, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test.serial('AsyncOperationS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + id: t.context.asyncOperations[2].id, + updatedAt: `${t.context.asyncOperationSearchTimestamp}`, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('AsyncOperationS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 100); + t.is(results?.length, 100); +}); + +test.serial('AsyncOperationS3Search returns fields specified', async (t) => { + const { connection } = t.context; + const fields = 'id,operationType,status,taskArn'; + const queryStringParameters = { + fields, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 100); + t.is(results?.length, 10); + results.forEach((asyncOperation) => t.deepEqual(Object.keys(asyncOperation), fields.split(','))); +}); + +test.serial('AsyncOperationS3Search supports sorting', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'id', + order: 'asc', + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].id < response.results[99].id); + t.true(response.results[0].id < response.results[50].id); + + queryStringParameters = { + limit: 200, + sort_key: ['-id'], + }; + const dbSearch2 = new AsyncOperationS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].id > response2.results[99].id); + t.true(response2.results[0].id > response2.results[50].id); + + queryStringParameters = { + limit: 200, + sort_by: 'operationType', + }; + const dbSearch3 = new AsyncOperationS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].operationType < response3.results[99].operationType); + t.true(response3.results[49].operationType < response3.results[50].operationType); +}); + +test.serial('AsyncOperationS3Search supports terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + operationType__in: ['Bulk Granules', 'NOTEXIST'].join(','), + }; + let dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + operationType__in: ['Bulk Granules', 'NOTEXIST'].join(','), + _id__in: [t.context.asyncOperations[2].id, uuidv4()].join(','), + }; + dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('AsyncOperationS3Search supports search when asyncOperation field does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + operationType__not: 'Bulk Granules', + }; + let dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + operationType__not: 'Bulk Granules', + id__not: t.context.asyncOperations[1].id, + }; + dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); +}); + +test.serial('AsyncOperationS3Search supports search which checks existence of asyncOperation field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + taskArn__exists: 'false', + output_exists: 'true', + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test.serial('AsyncOperationS3Search returns the correct record', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.asyncOperations[2]; + const queryStringParameters = { + limit: 200, + id: dbRecord.id, + }; + const dbSearch = new AsyncOperationS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + const expectedApiRecord = translatePostgresAsyncOperationToApiAsyncOperation(dbRecord); + t.deepEqual(omit(results?.[0], 'createdAt'), omit(expectedApiRecord, 'createdAt')); + t.truthy(results?.[0]?.createdAt); +}); diff --git a/packages/db/tests/s3search/test-CollectionS3Search.js b/packages/db/tests/s3search/test-CollectionS3Search.js new file mode 100644 index 000000000000..5d7a9795726b --- /dev/null +++ b/packages/db/tests/s3search/test-CollectionS3Search.js @@ -0,0 +1,556 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const omit = require('lodash/omit'); +const random = require('lodash/random'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { CollectionS3Search } = require('../../dist/s3search/CollectionS3Search'); +const { + collectionsS3TableSql, + granulesS3TableSql, + providersS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { translatePostgresCollectionToApiCollection } = require('../../dist/translate/collections'); + +const { + fakeCollectionRecordFactory, + fakeGranuleRecordFactory, + fakeProviderRecordFactory, +} = require('../../dist'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + t.context.collectionSearchTmestamp = 1579352700000; + + t.context.collections = range(100).map((num) => ( + fakeCollectionRecordFactory({ + name: num % 2 === 0 ? 'testCollection' : 'fakeCollection', + version: String(num), + cumulus_id: num, + updated_at: new Date(t.context.collectionSearchTmestamp + (num % 2)), + process: num % 2 === 0 ? 'ingest' : 'publish', + report_to_ems: num % 2 === 0, + url_path: num % 2 === 0 ? 'https://fakepath.com' : undefined, + granule_id_validation_regex: num % 2 === 0 ? 'testGranuleId' : 'fakeGranuleId', + }) + )); + + // Create provider + t.context.provider = fakeProviderRecordFactory({ + cumulus_id: random(1, 100), + }); + + const statuses = ['queued', 'failed', 'completed', 'running']; + t.context.granuleSearchTmestamp = 1688888800000; + t.context.granules = range(1000).map((num) => ( + fakeGranuleRecordFactory({ + // collection with cumulus_id 0-9 each has 11 granules, + // collection 10-98 has 10 granules, and collection 99 has 0 granule + collection_cumulus_id: num % 99, + cumulus_id: 100 + num, + // when collection_cumulus_id is odd number(1,3,5...97), its granules have provider + provider_cumulus_id: (num % 99 % 2) ? t.context.provider.cumulus_id : undefined, + status: statuses[num % 4], + // granule with collection_cumulus_id n has timestamp granuleSearchTmestamp + n, + // except granule 98 (with collection 98 ) which has timestamp granuleSearchTmestamp - 1 + updated_at: num === 98 + ? new Date(t.context.granuleSearchTmestamp - 1) + : new Date(t.context.granuleSearchTmestamp + (num % 99)), + }) + )); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'collections', + collectionsS3TableSql, + t.context.collections, + `${duckdbS3Prefix}collections.parquet` + ); + + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'providers', + providersS3TableSql, + t.context.provider, + `${duckdbS3Prefix}providers.parquet` + ); + + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'granules', + granulesS3TableSql, + t.context.granules, + `${duckdbS3Prefix}granules.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('CollectionS3Search returns 10 collections by default', async (t) => { + const { connection } = t.context; + const dbSearch = new CollectionS3Search({}, connection); + const results = await dbSearch.query(); + t.is(results.meta.count, 100); + t.is(results.results.length, 10); +}); + +test.serial('CollectionS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test.serial('CollectionS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 20, + infix: 'test', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 20); +}); + +test.serial('CollectionS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 20, + prefix: 'fake', + }; + const dbSearch2 = new CollectionS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 50); + t.is(response2.results?.length, 20); +}); + +test.serial('CollectionS3Search supports term search for boolean field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + reportToEms: 'false', + }; + const dbSearch4 = new CollectionS3Search({ queryStringParameters }, connection); + const response4 = await dbSearch4.query(); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); +}); + +test.serial('CollectionS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: `${t.context.collectionSearchTmestamp + 1}`, + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('CollectionS3Search supports term search for number field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + version: '2', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('CollectionS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + _id: 'fakeCollection___71', + }; + const dbSearch2 = new CollectionS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 1); + t.is(response2.results?.length, 1); + + queryStringParameters = { + limit: 200, + process: 'publish', + }; + const dbSearch3 = new CollectionS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 50); + t.is(response3.results?.length, 50); + + queryStringParameters = { + limit: 200, + granuleId: 'testGranuleId', + }; + const dbSearch4 = new CollectionS3Search({ queryStringParameters }, connection); + const response4 = await dbSearch4.query(); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); +}); + +test.serial('CollectionS3Search supports range search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + timestamp__from: `${t.context.collectionSearchTmestamp + 1}`, + timestamp__to: `${t.context.collectionSearchTmestamp + 2}`, + }; + let dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + ...queryStringParameters, + active: 'true', + }; + dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('CollectionS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + name: 'testCollection', + version: '0', + updatedAt: `${t.context.collectionSearchTmestamp}`, + process: 'ingest', + reportToEms: 'true', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('CollectionS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); +}); + +test.serial('CollectionS3Search returns fields specified', async (t) => { + const { connection } = t.context; + const fields = 'name,version,reportToEms,process'; + const queryStringParameters = { + fields, + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((collection) => t.deepEqual(Object.keys(collection), fields.split(','))); +}); + +test.serial('CollectionS3Search supports sorting', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'name', + order: 'asc', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].name < response.results[99].name); + t.true(response.results[0].name < response.results[50].name); + + queryStringParameters = { + limit: 200, + sort_key: ['-name'], + }; + const dbSearch2 = new CollectionS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].name > response2.results[99].name); + t.true(response2.results[0].name > response2.results[50].name); + + queryStringParameters = { + limit: 200, + sort_by: 'version', + }; + const dbSearch3 = new CollectionS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].version < response3.results[99].version); + t.true(response3.results[49].version < response3.results[50].version); +}); + +test.serial('CollectionS3Search supports terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + process__in: ['ingest', 'archive'].join(','), + }; + let dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + process__in: ['ingest', 'archive'].join(','), + _id__in: ['testCollection___0', 'fakeCollection___1'].join(','), + }; + dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 200, + granuleId__in: ['testGranuleId', 'non-existent'].join(','), + }; + dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('CollectionS3Search supports search when collection field does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + process__not: 'publish', + }; + let dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + process__not: 'publish', + version__not: 18, + }; + dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); +}); + +test.serial('CollectionS3Search supports search which checks existence of collection field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + url_path__exists: 'true', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('CollectionS3Search supports includeStats', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + includeStats: 'true', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + const expectedStats0 = { queued: 3, completed: 3, failed: 2, running: 3, total: 11 }; + const expectedStats98 = { queued: 2, completed: 3, failed: 3, running: 2, total: 10 }; + const expectedStats99 = { queued: 0, completed: 0, failed: 0, running: 0, total: 0 }; + + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[98].stats, expectedStats98); + t.deepEqual(response.results[99].stats, expectedStats99); +}); + +test.serial('CollectionS3Search supports search for active collections', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + const expectedStats0 = { queued: 3, completed: 3, failed: 2, running: 3, total: 11 }; + const expectedStats10 = { queued: 2, completed: 3, failed: 3, running: 2, total: 10 }; + const expectedStats98 = { queued: 2, completed: 3, failed: 3, running: 2, total: 10 }; + t.is(response.meta.count, 99); + t.is(response.results?.length, 99); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[10].stats, expectedStats10); + t.deepEqual(response.results[98].stats, expectedStats98); +}); + +test.serial('CollectionS3Search supports search for active collections by infix/prefix', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + infix: 'Collection', + prefix: 'fake', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + // collection_cumulus_id 1 + const expectedStats0 = { queued: 3, completed: 2, failed: 3, running: 3, total: 11 }; + // collection_cumulus_id 97 + const expectedStats48 = { queued: 3, completed: 2, failed: 3, running: 2, total: 10 }; + + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[48].stats, expectedStats48); +}); + +test.serial('CollectionS3Search support search for active collections and stats with granules updated in the given time frame', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + timestamp__from: `${t.context.granuleSearchTmestamp + 10}`, + timestamp__to: `${t.context.granuleSearchTmestamp + 98}`, + sort_by: 'version', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + const expectedStats0 = { queued: 2, completed: 3, failed: 3, running: 2, total: 10 }; + // collection with cumulus_id 98 has 9 granules in the time frame + const expectedStats98 = { queued: 2, completed: 2, failed: 3, running: 2, total: 9 }; + + // collections with cumulus_id 0-9 are filtered out + t.is(response.meta.count, 89); + t.is(response.results?.length, 89); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[88].stats, expectedStats98); +}); + +test.serial('CollectionS3Search support search for active collections and stats with granules from a given provider', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + provider: t.context.provider.name, + sort_by: 'version', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + // collection_cumulus_id 1 + const expectedStats0 = { queued: 3, completed: 2, failed: 3, running: 3, total: 11 }; + // collection_cumulus_id 97 + const expectedStats48 = { queued: 3, completed: 2, failed: 3, running: 2, total: 10 }; + + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[48].stats, expectedStats48); +}); + +test.serial('CollectionS3Search support search for active collections and stats with granules in the granuleId list', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + granuleId__in: [t.context.granules[0].granule_id, t.context.granules[5].granule_id].join(','), + sort_by: 'version', + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + // collection_cumulus_id 0 + const expectedStats0 = { queued: 1, completed: 0, failed: 0, running: 0, total: 1 }; + // collection_cumulus_id 5 + const expectedStats1 = { queued: 0, completed: 0, failed: 1, running: 0, total: 1 }; + + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[1].stats, expectedStats1); +}); + +test.serial('CollectionS3Search returns the correct record', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.collections[2]; + const queryStringParameters = { + limit: 200, + name: dbRecord.name, + version: dbRecord.version, + }; + const dbSearch = new CollectionS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + const expectedApiRecord = translatePostgresCollectionToApiCollection(dbRecord); + t.deepEqual(omit(results?.[0], 'createdAt'), omit(expectedApiRecord, 'createdAt')); + t.truthy(results?.[0]?.createdAt); +}); diff --git a/packages/db/tests/s3search/test-ExecutionS3Search.js b/packages/db/tests/s3search/test-ExecutionS3Search.js new file mode 100644 index 000000000000..2f72cf634126 --- /dev/null +++ b/packages/db/tests/s3search/test-ExecutionS3Search.js @@ -0,0 +1,699 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const omit = require('lodash/omit'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { constructCollectionId } = require('@cumulus/message/Collections'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + asyncOperationsS3TableSql, + collectionsS3TableSql, + executionsS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { ExecutionS3Search } = require('../../dist/s3search/ExecutionS3Search'); +const { + translatePostgresExecutionToApiExecutionWithoutDbQuery, +} = require('../../dist/translate/executions'); +const { + fakeAsyncOperationRecordFactory, + fakeCollectionRecordFactory, + fakeExecutionRecordFactory, +} = require('../../dist'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + const statuses = ['queued', 'failed', 'completed', 'running']; + const errors = [{ Error: 'UnknownError' }, { Error: 'CumulusMessageAdapterError' }, { Error: 'IngestFailure' }, { Error: 'CmrFailure' }, {}]; + + // Create a PG Collection + t.context.testPgCollection = fakeCollectionRecordFactory( + { cumulus_id: 0, + name: 'testCollection', + version: 8 } + ); + + t.context.collectionCumulusId = t.context.testPgCollection.cumulus_id; + + t.context.collectionId = constructCollectionId( + t.context.testPgCollection.name, + t.context.testPgCollection.version + ); + + t.context.testAsyncOperation = fakeAsyncOperationRecordFactory({ cumulus_id: 140 }); + t.context.asyncCumulusId = t.context.testAsyncOperation.cumulus_id; + + t.context.duration = 100; + + t.context.executions = range(50).map((num) => ( + fakeExecutionRecordFactory({ + collection_cumulus_id: num % 2 === 0 ? t.context.collectionCumulusId : undefined, + status: statuses[(num % 3) + 1], + error: errors[num % 5], + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + workflow_name: `testWorkflow__${num}`, + arn: num % 2 === 0 ? `testArn__${num}:testExecutionName` : `fakeArn__${num}:fakeExecutionName`, + url: `https://fake-execution${num}.com/`, + original_payload: { + orginal: `payload__${num}`, + }, + final_payload: num % 2 === 0 ? { + final: `payload__${num}`, + } : undefined, + duration: num > 0 ? t.context.duration * ((num % 2) + 1) : undefined, + async_operation_cumulus_id: num % 2 === 0 ? t.context.asyncCumulusId + : undefined, + parent_cumulus_id: num > 25 ? num % 25 : undefined, + cumulus_id: num, + timestamp: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + archived: Boolean(num % 2), + }) + )); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create collections'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'collections', + collectionsS3TableSql, + t.context.testPgCollection, + `${duckdbS3Prefix}collections.parquet` + ); + + console.log('create asyncOperation'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'async_operations', + asyncOperationsS3TableSql, + t.context.testAsyncOperation, + `${duckdbS3Prefix}async_operations.parquet` + ); + + console.log('create executions'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'executions', + executionsS3TableSql, + t.context.executions, + `${duckdbS3Prefix}executions.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('ExecutionS3Search returns correct response for basic query', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + estimateTableRowCount: 'false', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const results = await dbSearch.query(); + t.is(results.meta.count, 50); + t.is(results.results.length, 10); + const expectedResponse1 = { + name: 'testExecutionName', + status: 'failed', + arn: 'testArn__0:testExecutionName', + error: { Error: 'UnknownError' }, + originalPayload: { orginal: 'payload__0' }, + finalPayload: { final: 'payload__0' }, + type: 'testWorkflow__0', + execution: 'https://fake-execution0.com/', + collectionId: 'testCollection___8', + createdAt: new Date(2017, 11, 31).getTime(), + updatedAt: new Date(2018, 0, 1).getTime(), + timestamp: new Date(2018, 0, 1).getTime(), + archived: false, + }; + + const expectedResponse10 = { + name: 'fakeExecutionName', + status: 'failed', + arn: 'fakeArn__9:fakeExecutionName', + duration: 200, + error: {}, + originalPayload: { orginal: 'payload__9' }, + type: 'testWorkflow__9', + execution: 'https://fake-execution9.com/', + createdAt: new Date(2021, 9, 9).getTime(), + updatedAt: new Date(2021, 9, 10).getTime(), + timestamp: new Date(2021, 9, 10).getTime(), + archived: true, + }; + t.deepEqual(results.results[0], expectedResponse1); + t.deepEqual(results.results[9], expectedResponse10); +}); + +test.serial('ExecutionS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + estimateTableRowCount: 'false', + limit: 25, + page: 2, + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 10, + page: 5, + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 10, + page: 11, + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test.serial('ExecutionS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + infix: 'fake', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('ExecutionS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + prefix: 'test', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('ExecutionS3Search supports collectionId term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + collectionId: t.context.collectionId, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('ExecutionS3Search supports asyncOperationId term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + asyncOperationId: t.context.testAsyncOperation.id, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + t.is(response.results[0].asyncOperationId, t.context.testAsyncOperation.id); +}); + +test.serial('ExecutionS3Search supports term search for number field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + duration: 100, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 24); + t.is(response.results?.length, 24); +}); + +test.serial('ExecutionS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 50, + status: 'completed', + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 17); + t.is(response.results?.length, 17); + + queryStringParameters = { + limit: 50, + type: 'testWorkflow__5', + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('ExecutionS3Search supports term search for nested error.Error', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + 'error.Error': 'CumulusMessageAdapterError', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 10); + t.is(response.results?.length, 10); +}); + +test.serial('ExecutionS3Search supports range search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 50, + duration__from: 100, + duration__to: 150, + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 24); + t.is(response.results?.length, 24); + + queryStringParameters = { + limit: 200, + duration__from: 150, + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('ExecutionS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('ExecutionS3Search returns fields specified', async (t) => { + const { connection } = t.context; + const fields = 'status,arn,type,error'; + const queryStringParameters = { + estimateTableRowCount: 'false', + fields, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((execution) => t.deepEqual(Object.keys(execution), fields.split(','))); +}); + +test.serial('ExecutionS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + id: 13, + workflow_name: 'testWorkflow__13', + arn: 'fakeArn__13:fakeExecutionName', + url: 'https://fake-execution13.com/', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('ExecutionS3Search supports sorting', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + estimateTableRowCount: 'false', + limit: 50, + sort_by: 'timestamp', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].updatedAt < response.results[49].updatedAt); + t.true(response.results[1].updatedAt < response.results[25].updatedAt); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 50, + sort_by: 'timestamp', + order: 'desc', + }; + const dbSearch2 = new ExecutionS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 50); + t.is(response2.results?.length, 50); + t.true(response2.results[0].updatedAt > response2.results[49].updatedAt); + t.true(response2.results[1].updatedAt > response2.results[25].updatedAt); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_key: ['-timestamp'], + }; + const dbSearch3 = new ExecutionS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 50); + t.is(response3.results?.length, 50); + t.true(response3.results[0].updatedAt > response3.results[49].updatedAt); + t.true(response3.results[1].updatedAt > response3.results[25].updatedAt); +}); + +test.serial('ExecutionS3Search supports sorting by Error', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 50, + sort_by: 'error.Error', + }; + const dbSearch7 = new ExecutionS3Search({ queryStringParameters }, connection); + const response7 = await dbSearch7.query(); + t.is(response7.results[0].error.Error, 'CmrFailure'); + t.is(response7.results[49].error.Error, undefined); + + queryStringParameters = { + limit: 50, + sort_by: 'error.Error.keyword', + order: 'desc', + }; + const dbSearch10 = new ExecutionS3Search({ queryStringParameters }, connection); + const response10 = await dbSearch10.query(); + + // Unlike PostgreSQL sorting, the {} error appears last + t.is(response10.results[0].error.Error, 'UnknownError'); + t.is(response10.results[49].error.Error, undefined); +}); + +test.serial('ExecutionS3Search supports terms search', async (t) => { + const { connection } = t.context; + + let queryStringParameters = { + limit: 50, + type__in: ['testWorkflow__1', 'testWorkflow__2'].join(','), + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + + queryStringParameters = { + limit: 50, + type__in: ['testWorkflow__1', 'testWorkflow__2'].join(','), + status__in: 'running', + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('ExecutionS3Search supports parentArn term search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + parentArn: 'fakeArn__21:fakeExecutionName', + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + const expectedResponse = { + name: 'testExecutionName', + status: 'completed', + arn: 'testArn__46:testExecutionName', + duration: 100, + error: { Error: 'CumulusMessageAdapterError' }, + originalPayload: { orginal: 'payload__46' }, + finalPayload: { final: 'payload__46' }, + type: 'testWorkflow__46', + execution: 'https://fake-execution46.com/', + collectionId: 'testCollection___8', + parentArn: 'fakeArn__21:fakeExecutionName', + createdAt: new Date(2022, 10, 16).getTime(), + updatedAt: new Date(2022, 10, 18).getTime(), + timestamp: new Date(2022, 10, 18).getTime(), + archived: false, + }; + + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + t.deepEqual(response.results[0], expectedResponse); + queryStringParameters = { + limit: 50, + parentArn__exists: 'true', + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 24); + t.is(response.results?.length, 24); + queryStringParameters = { + limit: 50, + parentArn__in: ['fakeArn__21:fakeExecutionName', 'testArn__22:testExecutionName'].join(','), + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + queryStringParameters = { + limit: 50, + parentArn__not: 'testArn__2:testExecutionName', + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 23); + t.is(response.results?.length, 23); +}); + +test.serial('ExecutionS3Search supports error.Error terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 50, + 'error.Error__in': ['CumulusMessageAdapterError', 'UnknownError'].join(','), + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 20); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 50, + 'error.Error__in': 'unknownerror', + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('ExecutionS3Search supports search which checks existence of execution field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + duration__exists: 'true', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); +}); + +test.serial('ExecutionS3Search supports search which execution field does not match the given value', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + status__not: 'completed', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 33); + t.is(response.results?.length, 33); +}); + +test.serial('ExecutionS3Search supports term search for timestamp', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 50, + timestamp: `${(new Date(2023, 11, 7)).getTime()}`, + }; + let dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + queryStringParameters = { + limit: 200, + timestamp__from: `${(new Date(2019, 2, 21)).getTime()}`, + timestamp__to: `${(new Date(2027, 1, 23)).getTime()}`, + }; + dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 36); + t.is(response.results?.length, 36); +}); + +test.serial('ExecutionS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + updatedAt: `${new Date(2018, 0, 20).getTime()}`, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('ExecutionS3Search includeFullRecord', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + estimateTableRowCount: 'false', + limit: 50, + includeFullRecord: 'true', + }; + + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const results = await dbSearch.query(); + t.is(results.meta.count, 50); + t.is(results.results.length, 50); + const expectedResponse1 = { + name: 'testExecutionName', + status: 'failed', + arn: 'testArn__0:testExecutionName', + error: { Error: 'UnknownError' }, + originalPayload: { orginal: 'payload__0' }, + finalPayload: { final: 'payload__0' }, + type: 'testWorkflow__0', + execution: 'https://fake-execution0.com/', + asyncOperationId: t.context.testAsyncOperation.id, + collectionId: 'testCollection___8', + createdAt: new Date(2017, 11, 31).getTime(), + updatedAt: new Date(2018, 0, 1).getTime(), + timestamp: new Date(2018, 0, 1).getTime(), + archived: false, + }; + + const expectedResponse40 = { + name: 'testExecutionName', + status: 'completed', + arn: 'testArn__40:testExecutionName', + duration: 100, + error: { Error: 'UnknownError' }, + originalPayload: { orginal: 'payload__40' }, + finalPayload: { final: 'payload__40' }, + type: 'testWorkflow__40', + execution: 'https://fake-execution40.com/', + asyncOperationId: t.context.testAsyncOperation.id, + collectionId: 'testCollection___8', + parentArn: 'fakeArn__15:fakeExecutionName', + createdAt: new Date(2022, 4, 10).getTime(), + updatedAt: new Date(2022, 4, 12).getTime(), + timestamp: new Date(2022, 4, 12).getTime(), + archived: false, + }; + + t.deepEqual(results.results[0], expectedResponse1); + t.deepEqual(results.results[40], expectedResponse40); +}); + +test.serial('ExecutionS3Search estimates the rowcount of the table by default', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.true(response.meta.count > 0, 'Expected response.meta.count to be greater than 0'); + t.is(response.results?.length, 50); +}); + +test.serial('ExecutionS3Search only returns count if countOnly is set to true', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + countOnly: 'true', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.true(response.meta.count > 0, 'Expected response.meta.count to be greater than 0'); + t.is(response.results?.length, 0); +}); + +test.serial('ExecutionS3Search with archived: true pulls only archive granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + archived: 'true', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + response.results.forEach((ExecutionRecord) => { + t.is(ExecutionRecord.archived, true); + }); +}); + +test.serial('ExecutionS3Search with archived: false pulls only non-archive granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + archived: 'false', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + response.results.forEach((ExecutionRecord) => { + t.is(ExecutionRecord.archived, false); + }); +}); + +test.serial('ExecutionS3Search returns the correct record', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.executions[2]; + const queryStringParameters = { + limit: 200, + arn: dbRecord.arn, + includeFullRecord: 'true', + }; + const dbSearch = new ExecutionS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + const expectedApiRecord = translatePostgresExecutionToApiExecutionWithoutDbQuery({ + executionRecord: dbRecord, + collectionId: t.context.collectionId, + asyncOperationId: t.context.testAsyncOperation.id, + parentArn: undefined, + }); + t.deepEqual(omit(results?.[0], 'createdAt'), omit(expectedApiRecord, 'createdAt')); + t.truthy(results?.[0]?.createdAt); +}); diff --git a/packages/db/tests/s3search/test-GranuleS3Search.js b/packages/db/tests/s3search/test-GranuleS3Search.js new file mode 100644 index 000000000000..6f15dc5e7ab5 --- /dev/null +++ b/packages/db/tests/s3search/test-GranuleS3Search.js @@ -0,0 +1,1243 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const omit = require('lodash/omit'); +const random = require('lodash/random'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { constructCollectionId } = require('@cumulus/message/Collections'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { GranuleS3Search } = require('../../dist/s3search/GranuleS3Search'); +const { + collectionsS3TableSql, + executionsS3TableSql, + filesS3TableSql, + granulesS3TableSql, + granulesExecutionsS3TableSql, + providersS3TableSql, + pdrsS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { + translatePostgresGranuleToApiGranuleWithoutDbQuery, +} = require('../../dist/translate/granules'); + +const { + fakeCollectionRecordFactory, + fakeGranuleRecordFactory, + fakePdrRecordFactory, + fakeProviderRecordFactory, + fakeFileRecordFactory, + fakeExecutionRecordFactory, +} = require('../../dist'); + +// generate granuleId for infix and prefix search +const generateGranuleId = (num) => { + let granuleId = cryptoRandomString({ length: 10 }); + if (num % 30 === 0) granuleId = `${cryptoRandomString({ length: 5 })}infix${cryptoRandomString({ length: 5 })}`; + if (num % 50 === 0) granuleId = `prefix${cryptoRandomString({ length: 10 })}`; + return granuleId; +}; + +// DuckDB float comparisons require a range to account for precision errors +const FLOAT_EPSILON = 1e-4; + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + // Create collection + t.context.collectionName = 'fakeCollection'; + t.context.collectionVersion = 'v1'; + + const collectionName2 = 'testCollection2'; + const collectionVersion2 = 'v2'; + + t.context.collectionId = constructCollectionId( + t.context.collectionName, + t.context.collectionVersion + ); + + t.context.collectionId2 = constructCollectionId( + collectionName2, + collectionVersion2 + ); + + t.context.testPgCollection = fakeCollectionRecordFactory({ + cumulus_id: random(1, 100), + name: t.context.collectionName, + version: t.context.collectionVersion, + }); + t.context.testPgCollection2 = fakeCollectionRecordFactory({ + cumulus_id: random(101, 200), + name: collectionName2, + version: collectionVersion2, + }); + + const collections = [ + t.context.testPgCollection, + t.context.testPgCollection2]; + + t.context.collectionCumulusId = t.context.testPgCollection.cumulus_id; + t.context.collectionCumulusId2 = t.context.testPgCollection2.cumulus_id; + + // Create provider + t.context.provider = fakeProviderRecordFactory({ + cumulus_id: random(1, 100), + }); + t.context.providerCumulusId = t.context.provider.cumulus_id; + + // Create PDR + t.context.pdr = fakePdrRecordFactory({ + cumulus_id: random(1, 100), + collection_cumulus_id: t.context.testPgCollection.cumulus_id, + provider_cumulus_id: t.context.providerCumulusId, + }); + t.context.pdrCumulusId = t.context.pdr.cumulus_id; + + // Create Granule + t.context.granuleSearchFields = { + beginningDateTime: '2020-03-16T19:50:24.757Z', + cmrLink: 'https://fakeLink', + duration: 6.8, + endingDateTime: '2020-03-17T10:00:00.000Z', + 'error.Error': 'CumulusMessageAdapterExecutionError', + lastUpdateDateTime: '2020-03-18T10:00:00.000Z', + processingEndDateTime: '2020-03-16T10:00:00.000Z', + 'queryFields.cnm.receivedTime': '2020-03-18T21:11:20.802Z', + 'queryFields.cnm.processCompleteTime': '2020-03-18T21:12:30.916Z', + 'queryFields.cnm.product.name': 'cnm_product_name', + productVolume: '6000', + timeToArchive: '700.29', + timeToPreprocess: '800.18', + status: 'failed', + timestamp: 1579352700000, + updatedAt: 1579352700000, + }; + + t.context.granuleIds = range(100).map(generateGranuleId); + + const error = { + Cause: 'cause string', + Error: 'CumulusMessageAdapterExecutionError', + }; + + t.context.granules = range(100) + .map((num) => fakeGranuleRecordFactory({ + cumulus_id: num, + granule_id: t.context.granuleIds[num], + collection_cumulus_id: (num % 2) + ? t.context.collectionCumulusId : t.context.collectionCumulusId2, + pdr_cumulus_id: !(num % 2) ? t.context.pdrCumulusId : undefined, + provider_cumulus_id: !(num % 2) ? t.context.providerCumulusId : undefined, + beginning_date_time: new Date(t.context.granuleSearchFields.beginningDateTime), + cmr_link: !(num % 100) ? t.context.granuleSearchFields.cmrLink : undefined, + duration: t.context.granuleSearchFields.duration + (num % 2), + ending_date_time: !(num % 2) + ? new Date(t.context.granuleSearchFields.endingDateTime) : new Date(), + error: !(num % 2) ? JSON.stringify(error) : undefined, + last_update_date_time: !(num % 2) + ? new Date(t.context.granuleSearchFields.lastUpdateDateTime) : undefined, + published: !!(num % 2), + product_volume: Math.round(Number(t.context.granuleSearchFields.productVolume) + * (1 / (num + 1))).toString(), + time_to_archive: !(num % 10) + ? Number(t.context.granuleSearchFields.timeToArchive) : undefined, + time_to_process: !(num % 20) + ? Number(t.context.granuleSearchFields.timeToPreprocess) : undefined, + status: !(num % 2) ? t.context.granuleSearchFields.status : 'completed', + updated_at: new Date(t.context.granuleSearchFields.timestamp + (num % 2) * 1000), + archived: Boolean(num % 2), + query_fields: num === 50 + ? { + cnm: { + receivedTime: t.context.granuleSearchFields['queryFields.cnm.receivedTime'], + processCompleteTime: + t.context.granuleSearchFields['queryFields.cnm.processCompleteTime'], + product: { + name: t.context.granuleSearchFields['queryFields.cnm.product.name'], + }, + }, + } + : undefined, + })); + + t.context.files = t.context.granules + .flatMap((granule, i) => [ + fakeFileRecordFactory( + { + cumulus_id: i, + granule_cumulus_id: granule.cumulus_id, + path: 'a.txt', + checksum_type: 'md5', + } + ), + fakeFileRecordFactory( + { + cumulus_id: i + 100, + granule_cumulus_id: granule.cumulus_id, + path: 'b.txt', + checksum_type: 'sha256', + } + ), + ]); + + const earlierBaseTime = Date.now(); + const earlierExecutionRecords = t.context.granules + .map((_, i) => fakeExecutionRecordFactory({ + cumulus_id: i + 1, + url: `earlierUrl${i}`, + created_at: new Date(earlierBaseTime + i), + updated_at: new Date(earlierBaseTime + i), + timestamp: new Date(earlierBaseTime + i), + })); + + const earlierGranuleExecutions = t.context.granules + .map((granule, i) => ({ + granule_cumulus_id: granule.cumulus_id, + execution_cumulus_id: earlierExecutionRecords[i].cumulus_id, + })); + + // it's important for later testing that these are uploaded strictly in order + const laterBaseTime = Date.now() + 100; + const laterExecutionRecords = range(100).map((i) => ( + fakeExecutionRecordFactory({ + cumulus_id: i + 200, + url: `laterUrl${i}`, + created_at: new Date(laterBaseTime + i), + updated_at: new Date(laterBaseTime + i), + timestamp: new Date(laterBaseTime + i), + }) + )); + + const laterGranuleExecutions = t.context.granules + .flatMap((granule, i) => ([ + { + granule_cumulus_id: granule.cumulus_id, + execution_cumulus_id: laterExecutionRecords[i].cumulus_id, + }, + { + granule_cumulus_id: granule.cumulus_id, + execution_cumulus_id: laterExecutionRecords[99 - i].cumulus_id, + }, + ])); + + const executions = [ + ...earlierExecutionRecords, + ...laterExecutionRecords, + ]; + + const granuleExecutions = [ + ...earlierGranuleExecutions, + ...laterGranuleExecutions, + ]; + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create collections'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'collections', + collectionsS3TableSql, + collections, + `${duckdbS3Prefix}collections.parquet` + ); + + console.log('create providers'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'providers', + providersS3TableSql, + t.context.provider, + `${duckdbS3Prefix}providers.parquet` + ); + + console.log('create granules'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'granules', + granulesS3TableSql, + t.context.granules, + `${duckdbS3Prefix}granules.parquet` + ); + + console.log('create files'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'files', + filesS3TableSql, + t.context.files, + `${duckdbS3Prefix}files.parquet` + ); + + console.log('create executions'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'executions', + executionsS3TableSql, + executions, + `${duckdbS3Prefix}executions.parquet` + ); + + console.log('create granules_executions'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'granules_executions', + granulesExecutionsS3TableSql, + granuleExecutions, + `${duckdbS3Prefix}granules_executions.parquet` + ); + + console.log('create pdrs'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'pdrs', + pdrsS3TableSql, + t.context.pdr, + `${duckdbS3Prefix}pdrs.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('GranuleS3Search returns 10 granule records by default', async (t) => { + const { connection } = t.context; + const dbSearch = new GranuleS3Search({}, connection); + const response = await dbSearch.query(); + + t.is(response.meta.count, 100); + + const apiGranules = response.results || {}; + t.is(apiGranules.length, 10); + const validatedRecords = apiGranules.filter((granule) => ( + [t.context.collectionId, t.context.collectionId2].includes(granule.collectionId) + && (!granule.provider || granule.provider === t.context.provider.name) + && (!granule.pdrName || granule.pdrName === t.context.pdr.name))); + t.is(validatedRecords.length, apiGranules.length); +}); + +test.serial('GranuleS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + estimateTableRowCount: 'false', + limit: 20, + page: 2, + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 11, + page: 10, + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 10, + page: 11, + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test.serial('GranuleS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + infix: 'infix', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 3); + t.is(response.results?.length, 3); +}); + +test.serial('GranuleS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + prefix: 'prefix', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); +}); + +test.serial('GranuleS3Search supports collectionId term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + collectionId: t.context.collectionId2, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports provider term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + provider: t.context.provider.name, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports pdrName term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + pdrName: t.context.pdr.name, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports term search for boolean field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + published: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + beginningDateTime: t.context.granuleSearchFields.beginningDateTime, + endingDateTime: t.context.granuleSearchFields.endingDateTime, + lastUpdateDateTime: t.context.granuleSearchFields.lastUpdateDateTime, + updatedAt: `${t.context.granuleSearchFields.updatedAt}`, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports term search for number field', async (t) => { + const { connection } = t.context; + + let queryStringParameters = { + limit: 5, + duration__from: `${t.context.granuleSearchFields.duration - FLOAT_EPSILON}`, + duration__to: `${t.context.granuleSearchFields.duration + FLOAT_EPSILON}`, + productVolume: t.context.granuleSearchFields.productVolume, + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 200, + timeToArchive__from: `${t.context.granuleSearchFields.timeToArchive - FLOAT_EPSILON}`, + timeToArchive__to: `${t.context.granuleSearchFields.timeToArchive + FLOAT_EPSILON}`, + timeToPreprocess__from: `${t.context.granuleSearchFields.timeToPreprocess - FLOAT_EPSILON}`, + timeToPreprocess__to: `${t.context.granuleSearchFields.timeToPreprocess + FLOAT_EPSILON}`, + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 5); + t.is(response.results?.length, 5); +}); + +test.serial('GranuleS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + status: t.context.granuleSearchFields.status, + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + cmrLink: t.context.granuleSearchFields.cmrLink, + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('GranuleS3Search supports term search for timestamp', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + timestamp: `${t.context.granuleSearchFields.timestamp}`, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports term search for nested error.Error', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + 'error.Error': 'CumulusMessageAdapterExecutionError', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports range search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + beginningDateTime__from: '2020-03-16', + duration__from: `${t.context.granuleSearchFields.duration - 1 - FLOAT_EPSILON}`, + duration__to: `${t.context.granuleSearchFields.duration + 1 + FLOAT_EPSILON}`, + timestamp__from: `${t.context.granuleSearchFields.timestamp}`, + timestamp__to: `${t.context.granuleSearchFields.timestamp + 1600}`, + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + + queryStringParameters = { + limit: 200, + timestamp__from: t.context.granuleSearchFields.timestamp, + timestamp__to: t.context.granuleSearchFields.timestamp + 500, + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + duration__from: `${t.context.granuleSearchFields.duration + 2}`, + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('GranuleS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId2, t.context.collectionId].join(','), + cmrLink__exists: 'false', + 'error.Error': t.context.granuleSearchFields['error.Error'], + provider: t.context.provider.name, + published__not: 'true', + status: 'failed', + timestamp__from: t.context.granuleSearchFields.timestamp, + timestamp__to: t.context.granuleSearchFields.timestamp + 500, + sort_key: ['collectionId', '-timestamp'], + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); +}); + +test.serial('GranuleS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); +}); + +test.serial('GranuleS3Search returns fields specified', async (t) => { + const { connection } = t.context; + const fields = 'granuleId,endingDateTime,collectionId,published,status'; + const queryStringParameters = { + estimateTableRowCount: 'false', + fields, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((granule) => t.deepEqual(Object.keys(granule), fields.split(','))); +}); + +test.serial('GranuleS3Search supports sorting', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_by: 'timestamp', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].updatedAt < response.results[99].updatedAt); + t.true(response.results[1].updatedAt < response.results[50].updatedAt); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_by: 'timestamp', + order: 'desc', + }; + const dbSearch2 = new GranuleS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].updatedAt > response2.results[99].updatedAt); + t.true(response2.results[1].updatedAt > response2.results[50].updatedAt); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_key: ['-timestamp'], + }; + const dbSearch3 = new GranuleS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].updatedAt > response3.results[99].updatedAt); + t.true(response3.results[1].updatedAt > response3.results[50].updatedAt); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_key: ['+productVolume'], + }; + const dbSearch4 = new GranuleS3Search({ queryStringParameters }, connection); + const response4 = await dbSearch4.query(); + t.is(response4.meta.count, 100); + t.is(response4.results?.length, 100); + t.true(Number(response4.results[0].productVolume) < Number(response4.results[1].productVolume)); + t.true(Number(response4.results[98].productVolume) < Number(response4.results[99].productVolume)); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_key: ['-timestamp', '+productVolume'], + }; + const dbSearch5 = new GranuleS3Search({ queryStringParameters }, connection); + const response5 = await dbSearch5.query(); + t.is(response5.meta.count, 100); + t.is(response5.results?.length, 100); + t.true(response5.results[0].updatedAt > response5.results[99].updatedAt); + t.true(response5.results[1].updatedAt > response5.results[50].updatedAt); + t.true(Number(response5.results[1].productVolume) < Number(response5.results[99].productVolume)); + t.true(Number(response5.results[0].productVolume) < Number(response5.results[10].productVolume)); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_key: ['-timestamp'], + sort_by: 'timestamp', + order: 'asc', + }; + const dbSearch6 = new GranuleS3Search({ queryStringParameters }, connection); + const response6 = await dbSearch6.query(); + t.is(response6.meta.count, 100); + t.is(response6.results?.length, 100); + t.true(response6.results[0].updatedAt < response6.results[99].updatedAt); + t.true(response6.results[1].updatedAt < response6.results[50].updatedAt); +}); + +test.serial('GranuleS3Search supports sorting by CollectionId', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_by: 'collectionId', + order: 'asc', + }; + const dbSearch8 = new GranuleS3Search({ queryStringParameters }, connection); + const response8 = await dbSearch8.query(); + t.is(response8.meta.count, 100); + t.is(response8.results?.length, 100); + t.true(response8.results[0].collectionId < response8.results[99].collectionId); + t.true(response8.results[0].collectionId < response8.results[50].collectionId); + + queryStringParameters = { + estimateTableRowCount: 'false', + limit: 200, + sort_key: ['-collectionId'], + }; + const dbSearch9 = new GranuleS3Search({ queryStringParameters }, connection); + const response9 = await dbSearch9.query(); + t.is(response9.meta.count, 100); + t.is(response9.results?.length, 100); + t.true(response9.results[0].collectionId > response9.results[99].collectionId); + t.true(response9.results[0].collectionId > response9.results[50].collectionId); +}); + +test.serial('GranuleS3Search supports sorting by Error', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'error.Error', + }; + const dbSearch7 = new GranuleS3Search({ queryStringParameters }, connection); + const response7 = await dbSearch7.query(); + t.is(response7.results[0].error.Error, 'CumulusMessageAdapterExecutionError'); + t.is(response7.results[99].error, undefined); + + queryStringParameters = { + limit: 200, + sort_by: 'error.Error.keyword', + order: 'asc', + }; + const dbSearch10 = new GranuleS3Search({ queryStringParameters }, connection); + const response10 = await dbSearch10.query(); + t.is(response10.results[0].error.Error, 'CumulusMessageAdapterExecutionError'); + t.is(response10.results[99].error, undefined); +}); + +test.serial('GranuleS3Search supports terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + granuleId__in: [t.context.granuleIds[0], t.context.granuleIds[5]].join(','), + published__in: 'true,false', + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + + queryStringParameters = { + limit: 200, + granuleId__in: [t.context.granuleIds[0], t.context.granuleIds[5]].join(','), + published__in: 'true', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('GranuleS3Search supports collectionId terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId2, constructCollectionId('fakecollectionterms', 'v1')].join(','), + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId, t.context.collectionId2].join(','), + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); +}); + +test.serial('GranuleS3Search supports provider terms search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + provider__in: [t.context.provider.name, 'fakeproviderterms'].join(','), + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports pdrName terms search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + pdrName__in: [t.context.pdr.name, 'fakepdrterms'].join(','), + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports error.Error terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + 'error.Error__in': [t.context.granuleSearchFields['error.Error'], 'unknownerror'].join(','), + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + 'error.Error__in': 'unknownerror', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('GranuleS3Search supports search when granule field does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + granuleId__not: t.context.granuleIds[0], + published__not: 'true', + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + + queryStringParameters = { + limit: 200, + granuleId__not: t.context.granuleIds[0], + published__not: 'false', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which collectionId does not match the given value', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + collectionId__not: t.context.collectionId2, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which provider does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + provider__not: t.context.provider.name, + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 200, + provider__not: 'providernotexist', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which pdrName does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + pdrName__not: t.context.pdr.name, + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 200, + pdrName__not: 'pdrnotexist', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which error.Error does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + 'error.Error__not': t.context.granuleSearchFields['error.Error'], + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 200, + 'error.Error__not': 'unknownerror', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which checks existence of granule field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + cmrLink__exists: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('GranuleS3Search supports search which checks existence of collectionId', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + collectionId__exists: 'true', + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + queryStringParameters = { + limit: 200, + collectionId__exists: 'false', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('GranuleS3Search supports search which checks existence of provider', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + provider__exists: 'true', + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + provider__exists: 'false', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which checks existence of pdrName', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + pdrName__exists: 'true', + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + pdrName__exists: 'false', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search supports search which checks existence of error', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + error__exists: 'true', + }; + let dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + error__exists: 'false', + }; + dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search estimates the rowcount of the table by default', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.true(response.meta.count > 0, 'Expected response.meta.count to be greater than 0'); + t.is(response.results?.length, 50); +}); + +test.serial('GranuleS3Search only returns count if countOnly is set to true', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + countOnly: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.true(response.meta.count > 0, 'Expected response.meta.count to be greater than 0'); + t.is(response.results?.length, 0); +}); + +test.serial('GranuleS3Search with includeFullRecord true retrieves associated file objects for granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.results?.length, 100); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.files?.length, 2); + t.true('bucket' in granuleRecord.files[0]); + t.true('key' in granuleRecord.files[0]); + t.true('bucket' in granuleRecord.files[1]); + t.true('key' in granuleRecord.files[1]); + }); +}); +test.serial('GranuleS3Search with includeFullRecord true retrieves associated file translated to api key format', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.results?.length, 100); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.files?.length, 2); + t.true('bucket' in granuleRecord.files[0]); + t.true('key' in granuleRecord.files[0]); + t.true('checksumType' in granuleRecord.files[0]); + t.true('bucket' in granuleRecord.files[1]); + t.true('key' in granuleRecord.files[1]); + t.true('checksumType' in granuleRecord.files[1]); + }); +}); + +test.serial('GranuleS3Search with includeFullRecord true retrieves one associated Url object for granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.results?.length, 100); + response.results.forEach((granuleRecord) => { + t.true('execution' in granuleRecord); + }); +}); + +test.serial('GranuleS3Search with includeFullRecord true retrieves latest associated Url object for granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.results?.length, 100); + response.results.sort((a, b) => a.cumulus_id - b.cumulus_id); + // these executions are loaded from lowest to highest number + // but each granule is associated with multiple executions: + // earlierUrl${i}, laterUrl${i}, and laterUrl${99-i} + // hence `laterUrl${max(i, 99-i)}` is the most recently updated execution + response.results.forEach((granuleRecord, i) => { + t.is(granuleRecord.execution, `laterUrl${Math.max(i, 99 - i)}`); + }); +}); + +test.serial('GranuleS3Search with includeFullRecord true retrieves granules, files and executions, with limit specifying number of granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 4, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.results?.length, 4); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.files?.length, 2); + t.true('bucket' in granuleRecord.files[0]); + t.true('key' in granuleRecord.files[0]); + t.true('bucket' in granuleRecord.files[1]); + t.true('key' in granuleRecord.files[1]); + }); +}); + +test.serial('GranuleS3Search with archived: true pulls only archive granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + archived: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.archived, true); + }); +}); + +test.serial('GranuleS3Search with archived: false pulls only non-archive granules', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + archived: 'false', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.archived, false); + }); +}); + +test.serial('GranuleS3Search returns the correct record', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.granules[2]; + const queryStringParameters = { + limit: 200, + granuleId: dbRecord.granule_id, + duration__from: `${t.context.granuleSearchFields.duration - FLOAT_EPSILON}`, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + + const expectedApiRecord = translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord: dbRecord, + collectionPgRecord: t.context.testPgCollection2, + executionUrls: [{ url: 'laterUrl97' }], + files: t.context.files.filter((file) => file.granule_cumulus_id === dbRecord.cumulus_id), + pdr: t.context.pdr, + providerPgRecord: t.context.provider, + }); + t.deepEqual(omit(results?.[0], ['createdAt', 'duration']), omit(expectedApiRecord, ['createdAt', 'duration'])); + t.truthy(results?.[0]?.createdAt); +}); + +test.serial('GranuleS3Search supports term search on nested json field', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.granules[50]; + const queryStringParameters = { + limit: 200, + 'queryFields.cnm.receivedTime': t.context.granuleSearchFields['queryFields.cnm.receivedTime'], + 'queryFields.cnm.processCompleteTime': t.context.granuleSearchFields['queryFields.cnm.processCompleteTime'], + 'queryFields.cnm.product.name__not': 'abc', + includeFullRecord: 'true', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + + const expectedApiRecord = translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord: dbRecord, + collectionPgRecord: t.context.testPgCollection2, + executionUrls: [{ url: 'laterUrl50' }], + files: t.context.files.filter((file) => file.granule_cumulus_id === dbRecord.cumulus_id), + pdr: t.context.pdr, + providerPgRecord: t.context.provider, + }); + // float fields won't match exactly + const omitFields = ['createdAt', 'duration', 'timeToArchive']; + t.deepEqual(omit(results?.[0], omitFields), omit(expectedApiRecord, omitFields)); + t.truthy(results?.[0]?.createdAt); +}); + +test.serial('GranuleS3Search supports terms search on nested json field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + 'queryFields.cnm.product.name__in': [t.context.granuleSearchFields['queryFields.cnm.product.name'], 'abc'].join(','), + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('GranuleS3Search supports existence checks and sorting for nested JSON fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + 'queryFields.cnm.product.name__exists': 'true', + 'queryFields.cnm.product.random__exists': 'false', + sort_by: 'queryFields.cnm.product.name', + order: 'asc', + }; + const dbSearch = new GranuleS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); diff --git a/packages/db/tests/s3search/test-PdrS3Search.js b/packages/db/tests/s3search/test-PdrS3Search.js new file mode 100644 index 000000000000..7ac022af3481 --- /dev/null +++ b/packages/db/tests/s3search/test-PdrS3Search.js @@ -0,0 +1,820 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const omit = require('lodash/omit'); +const cryptoRandomString = require('crypto-random-string'); +const random = require('lodash/random'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { constructCollectionId } = require('@cumulus/message/Collections'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + collectionsS3TableSql, + executionsS3TableSql, + providersS3TableSql, + pdrsS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { PdrS3Search } = require('../../dist/s3search/PdrS3Search'); + +const { + fakeCollectionRecordFactory, + fakeExecutionRecordFactory, + fakePdrRecordFactory, + fakeProviderRecordFactory, +} = require('../../dist'); + +// generate PDR name for infix and prefix search +const generatePdrName = (num) => { + let name = cryptoRandomString({ length: 10 }); + if (num % 30 === 0) name = `${cryptoRandomString({ length: 5 })}infix${cryptoRandomString({ length: 5 })}`; + if (num % 25 === 0) name = `prefix${cryptoRandomString({ length: 10 })}`; + return name; +}; + +// DuckDB float comparisons require a range to account for precision errors +const FLOAT_EPSILON = 1e-4; + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + // Create collection + t.context.collectionName = 'fakeCollection'; + t.context.collectionVersion = 'v1'; + + const collectionName2 = 'testCollection2'; + const collectionVersion2 = 'v2'; + + t.context.collectionId = constructCollectionId( + t.context.collectionName, + t.context.collectionVersion + ); + + t.context.collectionId2 = constructCollectionId( + collectionName2, + collectionVersion2 + ); + + t.context.testPgCollection = fakeCollectionRecordFactory({ + cumulus_id: random(1, 100), + name: t.context.collectionName, + version: t.context.collectionVersion, + }); + t.context.testPgCollection2 = fakeCollectionRecordFactory({ + cumulus_id: random(101, 200), + name: collectionName2, + version: collectionVersion2, + }); + + t.context.collectionCumulusId = t.context.testPgCollection.cumulus_id; + t.context.collectionCumulusId2 = t.context.testPgCollection2.cumulus_id; + + // Create provider + t.context.provider = fakeProviderRecordFactory({ + cumulus_id: random(1, 100), + }); + + t.context.providerCumulusId = t.context.provider.cumulus_id; + + // Create execution + t.context.execution = fakeExecutionRecordFactory({ + cumulus_id: random(1, 100), + }); + + t.context.executionCumulusId = t.context.execution.cumulus_id; + + t.context.pdrSearchFields = { + createdAt: 1579352700000, + duration: 6.8, + progress: 0.9, + status: 'failed', + timestamp: 1579352700000, + updatedAt: 1579352700000, + }; + + t.context.pdrNames = range(100).map(generatePdrName); + t.context.pdrs = range(50).map((num) => fakePdrRecordFactory({ + cumulus_id: num, + name: t.context.pdrNames[num], + created_at: new Date(t.context.pdrSearchFields.createdAt), + collection_cumulus_id: (num % 2) + ? t.context.collectionCumulusId : t.context.collectionCumulusId2, + provider_cumulus_id: t.context.providerCumulusId, + execution_cumulus_id: !(num % 2) ? t.context.executionCumulusId : undefined, + status: !(num % 2) ? t.context.pdrSearchFields.status : 'completed', + progress: num / 50, + pan_sent: num % 2 === 0, + pan_message: `pan${cryptoRandomString({ length: 10 })}`, + stats: { + processing: 0, + completed: 0, + failed: 0, + total: 0, + }, + address: `address${cryptoRandomString({ length: 10 })}`, + original_url: !(num % 50) ? `url${cryptoRandomString({ length: 10 })}` : undefined, + duration: t.context.pdrSearchFields.duration + (num % 2), + updated_at: new Date(t.context.pdrSearchFields.timestamp + (num % 2) * 1000), + })); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create collections'); + const collections = [t.context.testPgCollection, t.context.testPgCollection2]; + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'collections', + collectionsS3TableSql, + collections, + `${duckdbS3Prefix}collections.parquet` + ); + + console.log('create providers'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'providers', + providersS3TableSql, + t.context.provider, + `${duckdbS3Prefix}providers.parquet` + ); + + console.log('create executions'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'executions', + executionsS3TableSql, + t.context.execution, + `${duckdbS3Prefix}executions.parquet` + ); + + console.log('create pdrs'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'pdrs', + pdrsS3TableSql, + t.context.pdrs, + `${duckdbS3Prefix}pdrs.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('PdrS3Search returns 10 PDR records by default', async (t) => { + const { connection } = t.context; + const execution = `https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/${t.context.execution.arn}`; + const dbSearch = new PdrS3Search({}, connection); + const response = await dbSearch.query(); + + t.is(response.meta.count, 50); + + const apiPdrs = response.results || {}; + t.is(apiPdrs.length, 10); + const validatedRecords = apiPdrs.filter((pdr) => ( + [t.context.collectionId, t.context.collectionId2].includes(pdr.collectionId) + && (pdr.provider === t.context.provider.name) + && (!pdr.execution || pdr.execution === execution))); + t.is(validatedRecords.length, apiPdrs.length); +}); + +test.serial('PdrS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 25, + page: 2, + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 10, + page: 5, + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test.serial('PdrS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + infix: 'infix', + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('PdrS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + prefix: 'prefix', + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); +}); + +test.serial('PdrS3Search supports collectionId term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + collectionId: t.context.collectionId2, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports provider term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + provider: t.context.provider.name, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('PdrS3Search supports execution term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + execution: `https://example.com/${t.context.execution.arn}`, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports term search for boolean field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + PANSent: 'true', + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + updatedAt: `${t.context.pdrSearchFields.updatedAt}`, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports term search for number field', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + duration__from: `${t.context.pdrSearchFields.duration - FLOAT_EPSILON}`, + duration__to: `${t.context.pdrSearchFields.duration + FLOAT_EPSILON}`, + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + progress__from: `${t.context.pdrSearchFields.progress - FLOAT_EPSILON}`, + progress__to: `${t.context.pdrSearchFields.progress + FLOAT_EPSILON}`, + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('PdrS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + status: t.context.pdrSearchFields.status, + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + const dbRecord = t.context.pdrs[0]; + queryStringParameters = { + limit: 100, + address: dbRecord.address, + pdrName: dbRecord.name, + originalUrl: dbRecord.original_url, + PANmessage: dbRecord.pan_message, + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('PdrS3Search supports term search for timestamp', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + timestamp: `${t.context.pdrSearchFields.timestamp}`, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports range search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + duration__from: `${t.context.pdrSearchFields.duration - 1 - FLOAT_EPSILON}`, + duration__to: `${t.context.pdrSearchFields.duration + 1 + FLOAT_EPSILON}`, + timestamp__from: `${t.context.pdrSearchFields.timestamp}`, + timestamp__to: `${t.context.pdrSearchFields.timestamp + 1600}`, + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 100, + timestamp__from: t.context.pdrSearchFields.timestamp, + timestamp__to: t.context.pdrSearchFields.timestamp + 500, + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + duration__from: `${t.context.pdrSearchFields.duration + 2}`, + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('PdrS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + collectionId__in: [t.context.collectionId2, t.context.collectionId].join(','), + provider: t.context.provider.name, + PANSent__not: 'false', + status: 'failed', + timestamp__from: t.context.pdrSearchFields.timestamp, + timestamp__to: t.context.pdrSearchFields.timestamp + 500, + sort_key: ['collectionId', '-timestamp'], + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('PdrS3Search returns fields specified', async (t) => { + const { connection } = t.context; + const fields = 'pdrName,collectionId,progress,PANSent,status'; + const queryStringParameters = { + fields, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((pdr) => t.deepEqual(Object.keys(pdr), fields.split(','))); +}); + +test.serial('PdrS3Search supports sorting', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + sort_by: 'timestamp', + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].updatedAt < response.results[25].updatedAt); + t.true(response.results[1].updatedAt < response.results[40].updatedAt); + + queryStringParameters = { + limit: 100, + sort_by: 'timestamp', + order: 'desc', + }; + const dbSearch2 = new PdrS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 50); + t.is(response2.results?.length, 50); + t.true(response2.results[0].updatedAt > response2.results[25].updatedAt); + t.true(response2.results[1].updatedAt > response2.results[40].updatedAt); + + queryStringParameters = { + limit: 100, + sort_key: ['-timestamp'], + }; + const dbSearch3 = new PdrS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 50); + t.is(response3.results?.length, 50); + t.true(response3.results[0].updatedAt > response3.results[25].updatedAt); + t.true(response3.results[1].updatedAt > response3.results[40].updatedAt); + + queryStringParameters = { + limit: 100, + sort_key: ['+progress'], + }; + const dbSearch4 = new PdrS3Search({ queryStringParameters }, connection); + const response4 = await dbSearch4.query(); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); + t.true(Number(response4.results[0].progress) < Number(response4.results[25].progress)); + t.true(Number(response4.results[1].progress) < Number(response4.results[40].progress)); + + queryStringParameters = { + limit: 100, + sort_key: ['-timestamp', '+progress'], + }; + const dbSearch5 = new PdrS3Search({ queryStringParameters }, connection); + const response5 = await dbSearch5.query(); + t.is(response5.meta.count, 50); + t.is(response5.results?.length, 50); + t.true(response5.results[0].updatedAt > response5.results[25].updatedAt); + t.true(response5.results[1].updatedAt > response5.results[40].updatedAt); + t.true(Number(response5.results[0].progress) < Number(response5.results[10].progress)); + t.true(Number(response5.results[30].progress) < Number(response5.results[40].progress)); + + queryStringParameters = { + limit: 100, + sort_key: ['-timestamp'], + sort_by: 'timestamp', + order: 'asc', + }; + const dbSearch6 = new PdrS3Search({ queryStringParameters }, connection); + const response6 = await dbSearch6.query(); + t.is(response6.meta.count, 50); + t.is(response6.results?.length, 50); + t.true(response6.results[0].updatedAt < response6.results[25].updatedAt); + t.true(response6.results[1].updatedAt < response6.results[40].updatedAt); +}); + +test.serial('PdrS3Search supports sorting by CollectionId', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + sort_by: 'collectionId', + order: 'asc', + }; + const dbSearch8 = new PdrS3Search({ queryStringParameters }, connection); + const response8 = await dbSearch8.query(); + t.is(response8.meta.count, 50); + t.is(response8.results?.length, 50); + t.true(response8.results[0].collectionId < response8.results[25].collectionId); + t.true(response8.results[1].collectionId < response8.results[40].collectionId); + + queryStringParameters = { + limit: 100, + sort_key: ['-collectionId'], + }; + const dbSearch9 = new PdrS3Search({ queryStringParameters }, connection); + const response9 = await dbSearch9.query(); + t.is(response9.meta.count, 50); + t.is(response9.results?.length, 50); + t.true(response9.results[0].collectionId > response9.results[25].collectionId); + t.true(response9.results[1].collectionId > response9.results[40].collectionId); +}); + +test.serial('PdrS3Search supports terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + pdrName__in: [t.context.pdrNames[0], t.context.pdrNames[5]].join(','), + PANSent__in: 'true,false', + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + + queryStringParameters = { + limit: 100, + pdrName__in: [t.context.pdrNames[0], t.context.pdrNames[5]].join(','), + PANSent__in: 'true', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('PdrS3Search supports collectionId terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + collectionId__in: [t.context.collectionId2, constructCollectionId('fakecollectionterms', 'v1')].join(','), + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + collectionId__in: [t.context.collectionId, t.context.collectionId2].join(','), + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('PdrS3Search supports provider terms search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + provider__in: [t.context.provider.name, 'fakeproviderterms'].join(','), + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('PdrS3Search supports execution terms search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + execution__in: [`https://example.con/${t.context.execution.arn}`, 'fakepdrterms'].join(','), + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports search when pdr field does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + pdrName__not: t.context.pdrNames[0], + PANSent__not: 'true', + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + pdrName__not: t.context.pdrNames[0], + PANSent__not: 'false', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 24); + t.is(response.results?.length, 24); +}); + +test.serial('PdrS3Search supports search which collectionId does not match the given value', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + collectionId__not: t.context.collectionId2, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports search which provider does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + provider__not: t.context.provider.name, + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 100, + provider__not: 'providernotexist', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('PdrS3Search supports search which execution does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + execution__not: `https://example.com/${t.context.execution.arn}`, + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 100, + execution__not: 'executionnotexist', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search supports search which checks existence of PDR field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + originalUrl__exists: 'true', + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('PdrS3Search supports search which checks existence of collectionId', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + collectionId__exists: 'true', + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + queryStringParameters = { + limit: 100, + collectionId__exists: 'false', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('PdrS3Search supports search which checks existence of provider', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + provider__exists: 'true', + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 100, + provider__exists: 'false', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test.serial('PdrS3Search supports search which checks existence of execution', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 100, + execution__exists: 'true', + }; + let dbSearch = new PdrS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + execution__exists: 'false', + }; + dbSearch = new PdrS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('PdrS3Search returns the correct record', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.pdrs[2]; + const queryStringParameters = { + limit: 100, + pdrName: dbRecord.name, + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + + const expectedApiRecord = { + pdrName: dbRecord.name, + provider: t.context.provider.name, + collectionId: t.context.collectionId2, + status: dbRecord.status, + createdAt: dbRecord.created_at.getTime(), + progress: dbRecord.progress, + execution: `https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/${t.context.execution.arn}`, + PANSent: dbRecord.pan_sent, + PANmessage: dbRecord.pan_message, + stats: { total: 0, failed: 0, completed: 0, processing: 0 }, + address: dbRecord.address, + duration: dbRecord.duration, + updatedAt: dbRecord.updated_at.getTime(), + }; + + t.deepEqual( + omit(results?.[0], ['duration', 'progress']), + omit(expectedApiRecord, ['duration', 'progress']) + ); + t.true( + Math.abs(results?.[0].duration - expectedApiRecord.duration) <= FLOAT_EPSILON + ); + + t.true( + Math.abs(results?.[0].progress - expectedApiRecord.progress) <= FLOAT_EPSILON + ); +}); + +test.serial('PdrS3Search only returns count if countOnly is set to true', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + countOnly: 'true', + }; + const dbSearch = new PdrS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.true(response.meta.count > 0, 'Expected response.meta.count to be greater than 0'); + t.is(response.results?.length, 0); +}); diff --git a/packages/db/tests/s3search/test-ProviderS3Search.js b/packages/db/tests/s3search/test-ProviderS3Search.js new file mode 100644 index 000000000000..f923f5190c9c --- /dev/null +++ b/packages/db/tests/s3search/test-ProviderS3Search.js @@ -0,0 +1,350 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const omit = require('lodash/omit'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + providersS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { ProviderS3Search } = require('../../dist/s3search/ProviderS3Search'); +const { + translatePostgresProviderToApiProvider, +} = require('../../dist/translate/providers'); +const { + fakeProviderRecordFactory, +} = require('../../dist'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + t.context.providerSearchTimestamp = 1579352700000; + + t.context.providers = range(100).map((num) => fakeProviderRecordFactory({ + cumulus_id: num, + updated_at: new Date(t.context.providerSearchTimestamp + (num % 2)), + created_at: new Date(t.context.providerSearchTimestamp - (num % 2)), + name: num % 2 === 0 ? `testProvider${num}` : `fakeProvider${num}`, + host: num % 2 === 0 ? 'cumulus-sit' : 'cumulus-uat', + global_connection_limit: num % 2 === 0 ? 0 : 10, + private_key: num % 2 === 0 ? `fakeKey${num}` : undefined, + })); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create providers'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'providers', + providersS3TableSql, + t.context.providers, + `${duckdbS3Prefix}providers.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('ProviderS3Search returns 10 providers by default', async (t) => { + const { connection } = t.context; + const dbSearch = new ProviderS3Search({}, connection); + const results = await dbSearch.query(); + t.is(results.meta.count, 100); + t.is(results.results.length, 10); +}); + +test.serial('ProviderS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test.serial('ProviderS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 20, + infix: 'test', + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 20); + t.true(response.results?.every((provider) => provider.id.includes('test'))); +}); + +test.serial('ProviderS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 20, + prefix: 'fake', + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 20); + t.true(response.results?.every((provider) => provider.id.startsWith('fake'))); +}); + +test.serial('ProviderS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const testUpdatedAt = t.context.providerSearchTimestamp + 1; + const queryStringParameters = { + limit: 200, + updatedAt: `${testUpdatedAt}`, + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.updatedAt === testUpdatedAt)); +}); + +test.serial('ProviderS3Search supports term search for number field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + globalConnectionLimit: '10', + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.globalConnectionLimit === 10)); +}); + +test.serial('ProviderS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + host: 'cumulus-sit', + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.host === 'cumulus-sit')); +}); + +test.serial('ProviderS3Search supports range search', async (t) => { + const { connection } = t.context; + const timestamp1 = t.context.providerSearchTimestamp + 1; + const timestamp2 = t.context.providerSearchTimestamp + 2; + const queryStringParameters = { + limit: 200, + timestamp__from: `${timestamp1}`, + timestamp__to: `${timestamp2}`, + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.updatedAt >= timestamp1 + && provider.updatedAt <= timestamp2)); +}); + +test.serial('ProviderS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + id: 'testProvider82', + host: 'cumulus-sit', + global_connection_limit: 0, + }; + + const expectedResponse = { + createdAt: 1579352700000, + host: 'cumulus-sit', + id: 'testProvider82', + globalConnectionLimit: 0, + privateKey: 'fakeKey82', + protocol: 's3', + updatedAt: 1579352700000, + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + t.deepEqual(response.results[0], expectedResponse); +}); + +test.serial('ProviderS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); +}); + +test.serial('ProviderS3Search returns fields specified', async (t) => { + const { connection } = t.context; + let fields = 'id'; + let queryStringParameters = { + fields, + }; + let dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((provider) => t.deepEqual(Object.keys(provider), fields.split(','))); + + fields = 'id,host,globalConnectionLimit'; + queryStringParameters = { + fields, + }; + dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((provider) => t.deepEqual(Object.keys(provider), fields.split(','))); +}); + +test.serial('ProviderS3Search supports sorting', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'id', + order: 'asc', + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].id < response.results[99].id); + t.true(response.results[0].id < response.results[50].id); + + queryStringParameters = { + limit: 200, + sort_key: ['-id'], + }; + const dbSearch2 = new ProviderS3Search({ queryStringParameters }, connection); + const response2 = await dbSearch2.query(); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].id > response2.results[99].id); + t.true(response2.results[0].id > response2.results[50].id); + + queryStringParameters = { + limit: 200, + sort_by: 'globalConnectionLimit', + }; + const dbSearch3 = new ProviderS3Search({ queryStringParameters }, connection); + const response3 = await dbSearch3.query(); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].globalConnectionLimit < response3.results[99].globalConnectionLimit); + t.true(response3.results[49].globalConnectionLimit < response3.results[50].globalConnectionLimit); +}); + +test.serial('ProviderS3Search supports terms search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + id__in: ['fakeProvider85', 'testProvider86'].join(','), + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + t.true(response.results?.every((provider) => ['fakeProvider85', 'testProvider86'].includes(provider.id))); +}); + +test.serial('ProviderS3Search supports search when provider field does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + host__not: 'cumulus-uat', + }; + let dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.host !== 'cumulus-uat')); + + queryStringParameters = { + limit: 200, + host__not: 'cumulus-uat', + id__not: 'testProvider38', + }; + dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + t.true(response.results?.every((provider) => provider.host !== 'cumulus-uat' && provider.id !== 'testProvider38')); +}); + +test.serial('ProviderS3Search supports search which checks existence of provider field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + privateKey__exists: 'true', + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.privateKey)); +}); + +test.serial('ProviderS3Search returns the correct record', async (t) => { + const { connection } = t.context; + const dbRecord = t.context.providers[2]; + const queryStringParameters = { + limit: 200, + name: dbRecord.name, + }; + const dbSearch = new ProviderS3Search({ queryStringParameters }, connection); + const { results, meta } = await dbSearch.query(); + t.is(meta.count, 1); + t.is(results?.length, 1); + const expectedApiRecord = translatePostgresProviderToApiProvider(dbRecord); + t.deepEqual(omit(results?.[0], 'createdAt'), omit(expectedApiRecord, 'createdAt')); + t.truthy(results?.[0]?.createdAt); +}); diff --git a/packages/db/tests/s3search/test-ReconciliationReportS3Search.js b/packages/db/tests/s3search/test-ReconciliationReportS3Search.js new file mode 100644 index 000000000000..0e4e6585680e --- /dev/null +++ b/packages/db/tests/s3search/test-ReconciliationReportS3Search.js @@ -0,0 +1,261 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + reconciliationReportsS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { ReconciliationReportS3Search } = require('../../dist/s3search/ReconciliationReportS3Search'); +const { + fakeReconciliationReportRecordFactory, +} = require('../../dist'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + const reconReportTypes = ['Granule Inventory', 'Granule Not Found', 'Inventory', 'ORCA Backup']; + const reconReportStatuses = ['Generated', 'Pending', 'Failed']; + t.context.reconReportSearchTimestamp = 1704100000000; + t.context.reportBucket = cryptoRandomString({ length: 8 }); + t.context.reportKey = cryptoRandomString({ length: 8 }); + + const reconReports = range(50).map((num) => fakeReconciliationReportRecordFactory({ + cumulus_id: num, + name: `fakeReconReport-${num + 1}`, + type: reconReportTypes[num % 4], + status: reconReportStatuses[num % 3], + location: `s3://fakeBucket${t.context.reportBucket}/fakeKey${t.context.reportKey}`, + updated_at: new Date(t.context.reconReportSearchTimestamp + (num % 2)), + created_at: new Date(t.context.reconReportSearchTimestamp - (num % 2)), + })); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create reconciliation reports'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'reconciliation_reports', + reconciliationReportsS3TableSql, + reconReports, + `${duckdbS3Prefix}reconciliation_reports.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('ReconciliationReportS3Search returns the correct response for a basic query', async (t) => { + const { connection } = t.context; + const dbSearch = new ReconciliationReportS3Search({}, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results.length, 10); + + const expectedResponse1 = { + name: 'fakeReconReport-1', + type: 'Granule Inventory', + status: 'Generated', + location: `s3://fakeBucket${t.context.reportBucket}/fakeKey${t.context.reportKey}`, + updatedAt: t.context.reconReportSearchTimestamp, + createdAt: t.context.reconReportSearchTimestamp, + }; + + const expectedResponse10 = { + name: 'fakeReconReport-10', + type: 'Granule Not Found', + status: 'Generated', + location: `s3://fakeBucket${t.context.reportBucket}/fakeKey${t.context.reportKey}`, + updatedAt: t.context.reconReportSearchTimestamp + 1, + createdAt: t.context.reconReportSearchTimestamp - 1, + }; + + t.deepEqual(response.results[0], expectedResponse1); + t.deepEqual(response.results[9], expectedResponse10); +}); + +test.serial('ReconciliationReportS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 25, + page: 2, + }; + let dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 10, + page: 5, + }; + dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test.serial('ReconciliationReportS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + prefix: 'fakeReconReport-1', + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 11); + t.is(response.results?.length, 11); +}); + +test.serial('ReconciliationReportS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + infix: 'conReport-2', + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 11); + t.is(response.results?.length, 11); +}); + +test.serial('ReconciliationReportS3Search supports sorting', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + sort_by: 'type', + order: 'asc', + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].type < response.results[15].type); + t.true(response.results[16].type < response.results[30].type); + t.true(response.results[31].type < response.results[45].type); +}); + +test.serial('ReconciliationReportS3Search supports term search for string fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + status: 'Generated', + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 17); + t.is(response.results?.length, 17); + t.true(response.results?.every((result) => result.status === 'Generated')); +}); + +test.serial('ReconciliationReportS3Search supports term search for date fields', async (t) => { + const { connection } = t.context; + const testUpdatedAt = t.context.reconReportSearchTimestamp + 1; + const queryStringParameters = { + limit: 100, + updatedAt: `${testUpdatedAt}`, + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + t.true(response.results?.every((report) => report.updatedAt === testUpdatedAt)); +}); + +test.serial('ReconciliationReportS3Search supports range search', async (t) => { + const { connection } = t.context; + const timestamp1 = t.context.reconReportSearchTimestamp - 1; + const timestamp2 = t.context.reconReportSearchTimestamp + 1; + const queryStringParameters = { + limit: 100, + timestamp__from: `${timestamp1}`, + timestamp__to: `${timestamp2}`, + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((report) => report.updatedAt >= timestamp1 + && report.updatedAt <= timestamp2)); +}); + +test.serial('ReconciliationReportS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + type: 'Inventory', + status: 'Failed', + }; + + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 4); + t.is(response.results?.length, 4); + t.true(response.results?.every((report) => + report.type === 'Inventory' && report.status === 'Failed')); +}); + +test.serial('ReconciliationReportS3Search returns fields specified', async (t) => { + const { connection } = t.context; + let fields = 'name'; + let queryStringParameters = { + fields, + }; + let dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((report) => t.deepEqual(Object.keys(report), fields.split(','))); + + fields = 'name,type,status'; + queryStringParameters = { + fields, + }; + dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((report) => t.deepEqual(Object.keys(report), fields.split(','))); +}); + +test.serial('ReconciliationReportS3Search ignores non-existing fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 100, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new ReconciliationReportS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); diff --git a/packages/db/tests/s3search/test-RuleS3Search.js b/packages/db/tests/s3search/test-RuleS3Search.js new file mode 100644 index 000000000000..f3db139f3cc4 --- /dev/null +++ b/packages/db/tests/s3search/test-RuleS3Search.js @@ -0,0 +1,440 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const random = require('lodash/random'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { constructCollectionId } = require('@cumulus/message/Collections'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + collectionsS3TableSql, + providersS3TableSql, + rulesS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); +const { RuleS3Search } = require('../../dist/s3search/RuleS3Search'); +const { + fakeCollectionRecordFactory, + fakeProviderRecordFactory, + fakeRuleRecordFactory, +} = require('../../dist'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + // Create PG Collections + t.context.testPgCollection = fakeCollectionRecordFactory( + { cumulus_id: 0, + name: 'testCollection', + version: 8 } + ); + t.context.testPgCollection2 = fakeCollectionRecordFactory( + { cumulus_id: 1, + name: 'testCollection2', + version: 4 } + ); + + t.context.collectionCumulusId = t.context.testPgCollection.cumulus_id; + t.context.collectionCumulusId2 = t.context.testPgCollection2.cumulus_id; + + t.context.collectionId = constructCollectionId( + t.context.testPgCollection.name, + t.context.testPgCollection.version + ); + t.context.collectionId2 = constructCollectionId( + t.context.testPgCollection2.name, + t.context.testPgCollection2.version + ); + + // Create a Provider + t.context.testProvider = fakeProviderRecordFactory({ + cumulus_id: random(1, 100), + name: 'testProvider', + }); + t.context.testProvider2 = fakeProviderRecordFactory({ + cumulus_id: random(101, 200), + name: 'testProvider2', + }); + + t.context.providerCumulusId = t.context.testProvider.cumulus_id; + t.context.providerCumulusId2 = t.context.testProvider2.cumulus_id; + + t.context.duration = 100; + + // Create a lot of Rules + t.context.ruleSearchFields = { + createdAt: new Date(2017, 11, 31), + updatedAt: new Date(2018, 0, 1), + updatedAt2: new Date(2018, 0, 2), + }; + + const rules = range(50).map((num) => fakeRuleRecordFactory({ + cumulus_id: num, + name: `fakeRule-${num}`, + created_at: t.context.ruleSearchFields.createdAt, + updated_at: (num % 2) ? + t.context.ruleSearchFields.updatedAt : t.context.ruleSearchFields.updatedAt2, + enabled: num % 2 === 0, + workflow: `testWorkflow-${num}`, + queue_url: (num % 2) ? 'https://sqs.us-east-1.amazonaws.com/123/456' : null, + collection_cumulus_id: (num % 2) + ? t.context.collectionCumulusId : t.context.collectionCumulusId2, + provider_cumulus_id: (num % 2) + ? t.context.providerCumulusId : t.context.providerCumulusId2, + })); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create collections'); + const collections = [t.context.testPgCollection, t.context.testPgCollection2]; + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'collections', + collectionsS3TableSql, + collections, + `${duckdbS3Prefix}collections.parquet` + ); + + console.log('create providers'); + const providers = [t.context.testProvider, t.context.testProvider2]; + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'providers', + providersS3TableSql, + providers, + `${duckdbS3Prefix}providers.parquet` + ); + + console.log('create rules'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'rules', + rulesS3TableSql, + rules, + `${duckdbS3Prefix}rules.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('RuleS3Search returns the correct response for a basic query', async (t) => { + const { connection } = t.context; + const dbSearch = new RuleS3Search({}, connection); + const results = await dbSearch.query(); + t.is(results.meta.count, 50); + t.is(results.results.length, 10); + + const expectedResponse1 = { + name: 'fakeRule-0', + createdAt: t.context.ruleSearchFields.createdAt.getTime(), + updatedAt: t.context.ruleSearchFields.updatedAt2.getTime(), + state: 'ENABLED', + rule: { + type: 'onetime', + }, + workflow: 'testWorkflow-0', + collection: { + name: 'testCollection2', + version: '4', + }, + provider: t.context.testProvider2.name, + }; + + const expectedResponse10 = { + name: 'fakeRule-9', + createdAt: t.context.ruleSearchFields.createdAt.getTime(), + updatedAt: t.context.ruleSearchFields.updatedAt.getTime(), + state: 'DISABLED', + rule: { + type: 'onetime', + }, + workflow: 'testWorkflow-9', + collection: { + name: 'testCollection', + version: '8', + }, + provider: t.context.testProvider.name, + queueUrl: 'https://sqs.us-east-1.amazonaws.com/123/456', + }; + + t.deepEqual(results.results[0], expectedResponse1); + t.deepEqual(results.results[9], expectedResponse10); +}); + +test.serial('RuleS3Search supports page and limit params', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 25, + page: 2, + }; + let dbSearch = new RuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 10, + page: 5, + }; + dbSearch = new RuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new RuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test.serial('RuleS3Search supports infix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + infix: 'Rule-27', + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('RuleS3Search supports prefix search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 50, + prefix: 'fakeRule-1', + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 11); + t.is(response.results?.length, 11); +}); + +test.serial('RuleS3Search supports term search for string field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 10, + workflow: 'testWorkflow-11', + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test.serial('RuleS3Search non-existing fields are ignored', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('RuleS3Search returns fields specified', async (t) => { + const { connection } = t.context; + const fields = 'state,name'; + const queryStringParameters = { + fields, + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((rule) => t.deepEqual(Object.keys(rule), fields.split(','))); +}); + +test.serial('RuleS3Search supports search for multiple fields', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 10, + prefix: 'fakeRule-1', + state: 'DISABLED', + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + t.is(response.meta.count, 6); + t.is(response.results?.length, 6); +}); + +test.serial('RuleS3Search supports sorting', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + sort_by: 'workflow', + order: 'desc', + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].workflow > response.results[10].workflow); + t.true(response.results[1].workflow > response.results[30].workflow); +}); + +test.serial('RuleS3Search supports collectionId term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + collectionId: t.context.collectionId, + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('RuleS3Search supports provider term search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + provider: t.context.testProvider.name, + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('RuleS3Search supports term search for date field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: t.context.ruleSearchFields.updatedAt, + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('RuleS3Search supports term search for boolean field', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + state: 'ENABLED', // maps to the bool field "enabled" + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('RuleS3Search supports term search for timestamp', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + timestamp: t.context.ruleSearchFields.updatedAt, //maps to timestamp + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('RuleS3Search supports range search', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + limit: 200, + timestamp__from: t.context.ruleSearchFields.timestamp, + timestamp__to: t.context.ruleSearchFields.timestamp + 1600, + }; + const dbSearch = new RuleS3Search({ queryStringParameters }, connection); + const response = await dbSearch.query(); + + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('RuleS3Search supports search which checks existence of queue URL field', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + queueUrl__exists: 'true', + }; + let dbSearch = new RuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 200, + queueUrl__exists: 'false', + }; + dbSearch = new RuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test.serial('RuleS3Search supports collectionId terms search', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId2, constructCollectionId('fakecollectionterms', 'v1')].join(','), + }; + let dbSearch = new RuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId, t.context.collectionId2].join(','), + }; + dbSearch = new RuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test.serial('RuleS3Search supports search which provider does not match the given value', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + limit: 200, + provider__not: t.context.testProvider.name, + }; + let dbSearch = new RuleS3Search({ queryStringParameters }, connection); + let response = await dbSearch.query(); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 200, + provider__not: 'providernotexist', + }; + dbSearch = new RuleS3Search({ queryStringParameters }, connection); + response = await dbSearch.query(); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); diff --git a/packages/db/tests/s3search/test-StatsS3Search.js b/packages/db/tests/s3search/test-StatsS3Search.js new file mode 100644 index 000000000000..1d184a76c08b --- /dev/null +++ b/packages/db/tests/s3search/test-StatsS3Search.js @@ -0,0 +1,488 @@ +'use strict'; + +const test = require('ava'); +const knex = require('knex'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); +const { s3 } = require('@cumulus/aws-client/services'); +const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); +const { + createDuckDBTables, + setupDuckDBWithS3ForTesting, + stageAndLoadDuckDBTableFromData, +} = require('../../dist/test-duckdb-utils'); +const { + collectionsS3TableSql, + executionsS3TableSql, + granulesS3TableSql, + providersS3TableSql, + pdrsS3TableSql, + reconciliationReportsS3TableSql, +} = require('../../dist/s3search/s3TableSchemas'); + +const { + fakeCollectionRecordFactory, + fakeGranuleRecordFactory, + fakePdrRecordFactory, + fakeProviderRecordFactory, + fakeExecutionRecordFactory, + fakeReconciliationReportRecordFactory, +} = require('../../dist'); + +const { StatsS3Search } = require('../../dist/s3search/StatsS3Search'); + +test.before(async (t) => { + t.context.knexBuilder = knex({ client: 'pg' }); + + const statuses = ['queued', 'failed', 'completed', 'running']; + const errors = [{ Error: 'UnknownError' }, { Error: 'CumulusMessageAdapterError' }, { Error: 'IngestFailure' }, { Error: 'CmrFailure' }, {}]; + const reconReportTypes = ['Granule Inventory', 'Granule Not Found', 'Inventory', 'ORCA Backup']; + const reconReportStatuses = ['Generated', 'Pending', 'Failed']; + + const collections = range(20).map((num) => fakeCollectionRecordFactory({ + name: 'testCollection', + version: `${num}`, + cumulus_id: num, + })); + + const providers = range(10).map((num) => fakeProviderRecordFactory({ + cumulus_id: num, + name: `testProvider${num}`, + })); + + const granules = range(100).map((num) => fakeGranuleRecordFactory({ + cumulus_id: num, + collection_cumulus_id: num % 20, + granule_id: num % 2 === 0 ? `testGranule${num}` : `query__Granule${num}`, + status: statuses[num % 4], + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + error: errors[num % 5], + duration: num + (num / 10), + provider_cumulus_id: num % 10, + })); + + const pdrs = range(20).map((num) => fakePdrRecordFactory({ + cumulus_id: num, + collection_cumulus_id: num, + status: statuses[(num % 3) + 1], + provider_cumulus_id: num % 10, + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + const executions = range(20).map((num) => fakeExecutionRecordFactory({ + cumulus_id: num, + collection_cumulus_id: num, + status: statuses[(num % 3) + 1], + error: errors[num % 5], + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + const reconReports = range(24).map((num) => fakeReconciliationReportRecordFactory({ + cumulus_id: num, + type: reconReportTypes[(num % 4)], + status: reconReportStatuses[(num % 3)], + created_at: (new Date(2024 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2024 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + const { instance, connection } = await setupDuckDBWithS3ForTesting(); + t.context.instance = instance; + t.context.connection = connection; + + t.context.testBucket = cryptoRandomString({ length: 10 }); + await s3().createBucket({ Bucket: t.context.testBucket }); + await createDuckDBTables(connection); + + const duckdbS3Prefix = `s3://${t.context.testBucket}/duckdb/`; + + console.log('create collections'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'collections', + collectionsS3TableSql, + collections, + `${duckdbS3Prefix}collections.parquet` + ); + + console.log('create providers'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'providers', + providersS3TableSql, + providers, + `${duckdbS3Prefix}providers.parquet` + ); + + console.log('create granules'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'granules', + granulesS3TableSql, + granules, + `${duckdbS3Prefix}granules.parquet` + ); + + console.log('create executions'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'executions', + executionsS3TableSql, + executions, + `${duckdbS3Prefix}executions.parquet` + ); + + console.log('create pdrs'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'pdrs', + pdrsS3TableSql, + pdrs, + `${duckdbS3Prefix}pdrs.parquet` + ); + + console.log('create reconciliation_reports'); + await stageAndLoadDuckDBTableFromData( + connection, + t.context.knexBuilder, + 'reconciliation_reports', + reconciliationReportsS3TableSql, + reconReports, + `${duckdbS3Prefix}reconciliation_reports.parquet` + ); +}); + +test.after.always(async (t) => { + await recursivelyDeleteS3Bucket(t.context.testBucket); + await t.context.connection.closeSync(); +}); + +test.serial('StatsS3Search aggregate returns correct response for basic query with type granules', async (t) => { + const { connection } = t.context; + const AggregateSearch = new StatsS3Search({}, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [ + { key: 'completed', count: 25 }, + { key: 'failed', count: 25 }, + { key: 'queued', count: 25 }, + { key: 'running', count: 25 }, + ]; + t.is(results.meta.count, 100); + t.deepEqual(results.count, expectedResponse); +}); + +test.serial('StatsS3Search aggregate filters granules correctly by date', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + timestamp__from: `${(new Date(2020, 1, 28)).getTime()}`, + timestamp__to: `${(new Date(2022, 2, 30)).getTime()}`, + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [ + { key: 'completed', count: 9 }, + { key: 'running', count: 9 }, + { key: 'failed', count: 8 }, + { key: 'queued', count: 8 }, + ]; + t.is(results.meta.count, 34); + t.deepEqual(results.count, expectedResponse); +}); + +test.serial('StatsS3Search aggregate filters executions correctly', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + field: 'status', + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'execution', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse1 = [ + { key: 'completed', count: 7 }, + { key: 'failed', count: 7 }, + { key: 'running', count: 6 }, + ]; + t.is(results.meta.count, 20); + t.deepEqual(results.count, expectedResponse1); + + queryStringParameters = { + field: 'status', + timestamp__to: `${(new Date(2023, 11, 30)).getTime()}`, + timestamp__from: `${(new Date(2021, 1, 28)).getTime()}`, + }; + + const AggregateSearch2 = new StatsS3Search({ queryStringParameters }, 'execution', connection); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [ + { key: 'completed', count: 3 }, + { key: 'failed', count: 3 }, + { key: 'running', count: 3 }, + ]; + t.is(results2.meta.count, 9); + t.deepEqual(results2.count, expectedResponse2); + + queryStringParameters = { + field: 'status', + timestamp__to: `${(new Date(2023, 11, 30)).getTime()}`, + timestamp__from: `${(new Date(2021, 1, 28)).getTime()}`, + collectionId: 'testCollection___5', + status: 'running', + }; + + const AggregateSearch3 = new StatsS3Search({ queryStringParameters }, 'execution', connection); + const results3 = await AggregateSearch3.aggregate(knex); + const expectedResponse3 = [{ key: 'running', count: 1 }]; + t.deepEqual(results3.count, expectedResponse3); + t.is(results3.meta.count, 1); +}); + +test.serial('StatsS3Search aggregate filters PDRs correctly', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + field: 'status', + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'pdr', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [ + { key: 'completed', count: 7 }, + { key: 'failed', count: 7 }, + { key: 'running', count: 6 }, + ]; + t.is(results.meta.count, 20); + t.deepEqual(results.count, expectedResponse); + + queryStringParameters = { + field: 'status', + timestamp__to: `${(new Date(2019, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2018, 1, 28)).getTime()}`, + }; + + const AggregateSearch2 = new StatsS3Search({ queryStringParameters }, 'pdr', connection); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [{ key: 'completed', count: 4 }, { key: 'failed', count: 2 }]; + t.is(results2.meta.count, 6); + t.deepEqual(results2.count, expectedResponse2); + + queryStringParameters = { + field: 'status', + timestamp__to: `${(new Date(2019, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2018, 1, 28)).getTime()}`, + status: 'failed', + }; + + const AggregateSearch3 = new StatsS3Search({ queryStringParameters }, 'pdr', connection); + const results3 = await AggregateSearch3.aggregate(knex); + const expectedResponse3 = [{ key: 'failed', count: 2 }]; + t.is(results3.meta.count, 2); + t.deepEqual(results3.count, expectedResponse3); +}); + +test.serial('StatsS3Search aggregate filters Reconciliation Reports correctly', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + field: 'type', + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'reconciliationReport', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [ + { key: 'Granule Inventory', count: 6 }, + { key: 'Granule Not Found', count: 6 }, + { key: 'Inventory', count: 6 }, + { key: 'ORCA Backup', count: 6 }, + ]; + t.is(results.meta.count, 24); + t.deepEqual(results.count, expectedResponse); + + queryStringParameters = { + field: 'status', + }; + + const AggregateSearch2 = new StatsS3Search({ queryStringParameters }, 'reconciliationReport', connection); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [ + { key: 'Failed', count: 8 }, + { key: 'Generated', count: 8 }, + { key: 'Pending', count: 8 }, + ]; + t.is(results2.meta.count, 24); + t.deepEqual(results2.count, expectedResponse2); +}); + +test.serial('StatsS3Search returns correct aggregate response for type granule when queried by provider', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + field: 'status', + provider: 'testProvider2', + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [{ key: 'completed', count: 5 }, { key: 'queued', count: 5 }]; + t.is(results.meta.count, 10); + t.deepEqual(results.count, expectedResponse); +}); + +test.serial('StatsS3Search returns correct aggregate response for type granule when queried by collection', async (t) => { + const { connection } = t.context; + const queryStringParameters = { + field: 'status', + collectionId: 'testCollection___8', + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [{ key: 'queued', count: 5 }]; + t.is(results.meta.count, 5); + t.deepEqual(results.count, expectedResponse); +}); + +test.serial('StatsS3Search returns correct aggregate response for type granule when queried by collection and provider', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + field: 'status', + collectionId: 'testCollection___1', + provider: 'testProvider1', + }; + + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [{ key: 'failed', count: 5 }]; + t.is(results.meta.count, 5); + t.deepEqual(results.count, expectedResponse); + + queryStringParameters = { + field: 'status', + collectionId: 'testCollection___1', + provider: 'testProvider1', + timestamp__to: `${(new Date(2019, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2018, 1, 28)).getTime()}`, + }; + + const AggregateSearch2 = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [{ key: 'failed', count: 2 }]; + t.is(results2.meta.count, 2); + t.deepEqual(results2.count, expectedResponse2); + queryStringParameters = { + field: 'status', + collectionId: 'testCollection___1', + provider: 'testProvider1', + timestamp__to: `${(new Date(2019, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2018, 1, 28)).getTime()}`, + status: 'failed', + }; + + const AggregateSearch3 = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results3 = await AggregateSearch3.aggregate(knex); + const expectedResponse3 = [{ key: 'failed', count: 2 }]; + t.is(results3.meta.count, 2); + t.deepEqual(results3.count, expectedResponse3); +}); + +test.serial('StatsS3Search returns correct aggregate response for type granule when queried by error', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + field: 'error.Error.keyword', + }; + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse1 = [ + { key: 'CmrFailure', count: 20 }, + { key: 'CumulusMessageAdapterError', count: 20 }, + { key: 'IngestFailure', count: 20 }, + { key: 'UnknownError', count: 20 }, + ]; + t.is(results.meta.count, 80); + t.deepEqual(results.count, expectedResponse1); + + queryStringParameters = { + field: 'error.Error.keyword', + timestamp__to: `${(new Date(2021, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2020, 1, 28)).getTime()}`, + }; + const AggregateSearch2 = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [ + { key: 'CmrFailure', count: 8 }, + { key: 'IngestFailure', count: 7 }, + { key: 'CumulusMessageAdapterError', count: 6 }, + { key: 'UnknownError', count: 6 }, + ]; + t.is(results2.meta.count, 27); + t.deepEqual(results2.count, expectedResponse2); + + queryStringParameters = { + collectionId: 'testCollection___1', + provider: 'testProvider1', + field: 'error.Error.keyword', + timestamp__to: `${(new Date(2019, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2018, 1, 28)).getTime()}`, + }; + const AggregateSearch3 = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results3 = await AggregateSearch3.aggregate(knex); + const expectedResponse3 = [{ key: 'CumulusMessageAdapterError', count: 2 }]; + t.is(results3.meta.count, 2); + t.deepEqual(results3.count, expectedResponse3); +}); + +test.serial('StatsS3Search can query by infix and prefix when type is defined', async (t) => { + const { connection } = t.context; + let queryStringParameters = { + infix: 'testGra', + }; + const AggregateSearch = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse1 = [{ key: 'completed', count: 25 }, { key: 'queued', count: 25 }]; + t.is(results.meta.count, 50); + t.deepEqual(results.count, expectedResponse1); + + queryStringParameters = { + prefix: 'query', + }; + const AggregateSearch2 = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [{ key: 'failed', count: 25 }, { key: 'running', count: 25 }]; + t.is(results2.meta.count, 50); + t.deepEqual(results2.count, expectedResponse2); + + queryStringParameters = { + infix: 'testCollection', + version: '8', + field: 'name', + }; + const AggregateSearch3 = new StatsS3Search({ queryStringParameters }, 'collection', connection); + const results3 = await AggregateSearch3.aggregate(knex); + const expectedResponse3 = [{ key: 'testCollection', count: 1 }]; + t.is(results3.meta.count, 1); + t.deepEqual(results3.count, expectedResponse3); +}); + +test.serial('StatsS3Search summary works', async (t) => { + const { connection } = t.context; + const StatsSummary = new StatsS3Search({}, 'granule', connection); + const results = await StatsSummary.summary(knex); + t.is(results.collections.value, 20); + t.is(results.granules.value, 100); + t.is(results.errors.value, 80); + t.is(results.processingTime.value, 54.44999999642372); + const queryStringParameters = { + timestamp__to: `${(new Date(2019, 12, 9)).getTime()}`, + timestamp__from: `${(new Date(2018, 1, 28)).getTime()}`, + }; + const StatsSummary2 = new StatsS3Search({ queryStringParameters }, 'granule', connection); + const results2 = await StatsSummary2.summary(knex); + t.is(results2.collections.value, 15); + t.is(results2.granules.value, 25); + t.is(results2.errors.value, 21); + t.is(results2.processingTime.value, 53.54799992084503); +}); diff --git a/packages/db/tests/search/test-ExecutionSearch.js b/packages/db/tests/search/test-ExecutionSearch.js index befdaa8b5677..4faf3b1b4aec 100644 --- a/packages/db/tests/search/test-ExecutionSearch.js +++ b/packages/db/tests/search/test-ExecutionSearch.js @@ -632,7 +632,7 @@ test('ExecutionSearch only returns count if countOnly is set to true', async (t) test('ExecutionSearch with archived: true pulls only archive granules', async (t) => { const { knex } = t.context; const queryStringParameters = { - archived: true, + archived: 'true', }; const dbSearch = new ExecutionSearch({ queryStringParameters }); const response = await dbSearch.query(knex); @@ -644,7 +644,7 @@ test('ExecutionSearch with archived: true pulls only archive granules', async (t test('ExecutionSearch with archived: false pulls only non-archive granules', async (t) => { const { knex } = t.context; const queryStringParameters = { - archived: false, + archived: 'false', }; const dbSearch = new ExecutionSearch({ queryStringParameters }); const response = await dbSearch.query(knex); diff --git a/packages/db/tests/search/test-GranuleSearch.js b/packages/db/tests/search/test-GranuleSearch.js index f43f4bf240b0..d1a8f5d58fe7 100644 --- a/packages/db/tests/search/test-GranuleSearch.js +++ b/packages/db/tests/search/test-GranuleSearch.js @@ -1,9 +1,13 @@ const test = require('ava'); const cryptoRandomString = require('crypto-random-string'); +const omit = require('lodash/omit'); const range = require('lodash/range'); const { constructCollectionId } = require('@cumulus/message/Collections'); const { sleep } = require('@cumulus/common'); +const { + translatePostgresGranuleToApiGranuleWithoutDbQuery, +} = require('../../dist/translate/granules'); const { CollectionPgModel, fakeCollectionRecordFactory, @@ -110,6 +114,9 @@ test.before(async (t) => { 'error.Error': 'CumulusMessageAdapterExecutionError', lastUpdateDateTime: '2020-03-18T10:00:00.000Z', processingEndDateTime: '2020-03-16T10:00:00.000Z', + 'queryFields.cnm.receivedTime': '2020-03-18T21:11:20.802Z', + 'queryFields.cnm.processCompleteTime': '2020-03-18T21:12:30.916Z', + 'queryFields.cnm.product.name': 'cnm_product_name', productVolume: '6000', timeToArchive: '700.29', timeToPreprocess: '800.18', @@ -126,9 +133,8 @@ test.before(async (t) => { }; t.context.granulePgModel = new GranulePgModel(); - t.context.pgGranules = await t.context.granulePgModel.insert( - knex, - range(100).map((num) => fakeGranuleRecordFactory({ + t.context.granules = range(100).map((num) => fakeGranuleRecordFactory( + { granule_id: t.context.granuleIds[num], collection_cumulus_id: (num % 2) ? t.context.collectionCumulusId : t.context.collectionCumulusId2, @@ -152,31 +158,45 @@ test.before(async (t) => { status: !(num % 2) ? t.context.granuleSearchFields.status : 'completed', updated_at: new Date(t.context.granuleSearchFields.timestamp + (num % 2) * 1000), archived: Boolean(num % 2), - })) - ); + query_fields: num === 50 + ? { + cnm: { + receivedTime: t.context.granuleSearchFields['queryFields.cnm.receivedTime'], + processCompleteTime: + t.context.granuleSearchFields['queryFields.cnm.processCompleteTime'], + product: { + name: t.context.granuleSearchFields['queryFields.cnm.product.name'], + }, + }, + } + : undefined, + } + )); + + t.context.pgGranules = await t.context.granulePgModel.insert(knex, t.context.granules); const filePgModel = new FilePgModel(); - await filePgModel.insert( - knex, - t.context.pgGranules.map((granule) => fakeFileRecordFactory( - { - granule_cumulus_id: granule.cumulus_id, - path: 'a.txt', - checksum_type: 'md5', - } - )) - ); - await filePgModel.insert( - knex, - t.context.pgGranules.map((granule) => fakeFileRecordFactory( - { - granule_cumulus_id: granule.cumulus_id, - path: 'b.txt', - checksum_type: 'sha256', - } - )) - ); - + t.context.files = t.context.pgGranules + .flatMap((granule, i) => [ + fakeFileRecordFactory( + { + cumulus_id: i, + granule_cumulus_id: granule.cumulus_id, + path: 'a.txt', + checksum_type: 'md5', + } + ), + fakeFileRecordFactory( + { + cumulus_id: i + 100, + granule_cumulus_id: granule.cumulus_id, + path: 'b.txt', + checksum_type: 'sha256', + } + ), + ]); + + await filePgModel.insert(knex, t.context.files); const executionPgModel = new ExecutionPgModel(); const granuleExecutionPgModel = new GranulesExecutionsPgModel(); @@ -1035,7 +1055,7 @@ test('GranuleSearch with includeFullRecord true retrieves granules, files and ex test('GranuleSearch with archived: true pulls only archive granules', async (t) => { const { knex } = t.context; const queryStringParameters = { - archived: true, + archived: 'true', }; const dbSearch = new GranuleSearch({ queryStringParameters }); const response = await dbSearch.query(knex); @@ -1047,7 +1067,7 @@ test('GranuleSearch with archived: true pulls only archive granules', async (t) test('GranuleSearch with archived: false pulls only non-archive granules', async (t) => { const { knex } = t.context; const queryStringParameters = { - archived: false, + archived: 'false', }; const dbSearch = new GranuleSearch({ queryStringParameters }); const response = await dbSearch.query(knex); @@ -1055,3 +1075,61 @@ test('GranuleSearch with archived: false pulls only non-archive granules', async t.is(granuleRecord.archived, false); }); }); + +test.serial('GranuleSearch supports term search on nested json field', async (t) => { + const { knex } = t.context; + const dbRecord = t.context.granules[50]; + const granuleCumulusId = t.context.pgGranules[50].cumulus_id; + const queryStringParameters = { + limit: 200, + 'queryFields.cnm.receivedTime': t.context.granuleSearchFields['queryFields.cnm.receivedTime'], + 'queryFields.cnm.processCompleteTime': t.context.granuleSearchFields['queryFields.cnm.processCompleteTime'], + 'queryFields.cnm.product.name__not': 'abc', + includeFullRecord: 'true', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); + + const expectedApiRecord = translatePostgresGranuleToApiGranuleWithoutDbQuery({ + granulePgRecord: dbRecord, + collectionPgRecord: t.context.testPgCollection2, + executionUrls: [{ url: 'laterUrl50' }], + files: t.context.files.filter((file) => file.granule_cumulus_id === granuleCumulusId), + pdr: t.context.pdr, + providerPgRecord: t.context.provider, + }); + + // float fields won't match exactly + const omitFields = ['createdAt', 'duration', 'timeToArchive']; + t.deepEqual(omit(results?.[0], omitFields), omit(expectedApiRecord, omitFields)); + t.truthy(results?.[0]?.createdAt); +}); + +test.serial('GranuleSearch supports terms search on nested json field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + 'queryFields.cnm.product.name__in': [t.context.granuleSearchFields['queryFields.cnm.product.name'], 'abc'].join(','), + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test.serial('GranuleSearch supports existence checks and sorting for nested JSON fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + 'queryFields.cnm.product.name__exists': 'true', + 'queryFields.cnm.product.random__exists': 'false', + sort_by: 'queryFields.cnm.product.name', + order: 'asc', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); diff --git a/packages/db/tests/translate/test-async-operations.js b/packages/db/tests/translate/test-async-operations.js index f824497cc721..cf8610469595 100644 --- a/packages/db/tests/translate/test-async-operations.js +++ b/packages/db/tests/translate/test-async-operations.js @@ -157,6 +157,13 @@ test('translatePostgresAsyncOperationToApiAsyncOperation translates PostgreSQL r description, }); + const duckDbAsyncOperation = { + ...pgAsyncOperation, + created_at: pgAsyncOperation.created_at.toISOString(), + updated_at: pgAsyncOperation.updated_at.toISOString(), + output: JSON.stringify(pgAsyncOperation.output), + }; + const expectedAsyncOperation = { id, status: 'RUNNING', @@ -169,4 +176,9 @@ test('translatePostgresAsyncOperationToApiAsyncOperation translates PostgreSQL r }; const translation = await translatePostgresAsyncOperationToApiAsyncOperation(pgAsyncOperation); t.deepEqual(translation, expectedAsyncOperation); + + const duckDbTranslation = await translatePostgresAsyncOperationToApiAsyncOperation( + duckDbAsyncOperation + ); + t.deepEqual(duckDbTranslation, expectedAsyncOperation); }); diff --git a/packages/db/tests/translate/test-collections.js b/packages/db/tests/translate/test-collections.js index 569763630c6c..383dee1aed31 100644 --- a/packages/db/tests/translate/test-collections.js +++ b/packages/db/tests/translate/test-collections.js @@ -18,10 +18,18 @@ test('translatePostgresCollectionToApiCollection converts Postgres collection to created_at: new Date(), }); + const duckDbCollectionRecord = { + ...collectionRecord, + created_at: collectionRecord.created_at.toISOString(), + updated_at: collectionRecord.updated_at.toISOString(), + meta: JSON.stringify(collectionRecord.meta), + tags: JSON.stringify(collectionRecord.tags), + }; + const expected = { createdAt: collectionRecord.created_at.getTime(), updatedAt: collectionRecord.updated_at.getTime(), - files: collectionRecord.files, + files: JSON.parse(collectionRecord.files), granuleId: collectionRecord.granule_id_validation_regex, granuleIdExtraction: collectionRecord.granule_id_validation_regex, meta: collectionRecord.meta, @@ -30,8 +38,11 @@ test('translatePostgresCollectionToApiCollection converts Postgres collection to tags: collectionRecord.tags, version: collectionRecord.version, }; - const actual = translatePostgresCollectionToApiCollection(collectionRecord); - t.deepEqual(actual, expected); + const translation = translatePostgresCollectionToApiCollection(collectionRecord); + t.deepEqual(translation, expected); + + const duckDbTranslation = translatePostgresCollectionToApiCollection(duckDbCollectionRecord); + t.deepEqual(duckDbTranslation, expected); }); test('translateApiCollectionToPostgresCollection converts API collection to Postgres', (t) => { diff --git a/packages/db/tests/translate/test-executions.js b/packages/db/tests/translate/test-executions.js index 9d1e8d0c1b90..95cb95219862 100644 --- a/packages/db/tests/translate/test-executions.js +++ b/packages/db/tests/translate/test-executions.js @@ -75,6 +75,16 @@ test('translatePostgresExecutionToApiExecution translates a Postgres execution t workflow_name: 'TestWorkflow', }; + const duckDbExecutionRecord = { + ...executionRecord, + created_at: executionRecord.created_at.toISOString(), + updated_at: executionRecord.updated_at.toISOString(), + timestamp: executionRecord.timestamp.toISOString(), + error: JSON.stringify(executionRecord.error), + final_payload: JSON.stringify(executionRecord.final_payload), + original_payload: JSON.stringify(executionRecord.original_payload), + }; + const expectedApiExecution = { arn: executionRecord.arn, asyncOperationId: 'asyncOperationCumulusId', @@ -95,7 +105,7 @@ test('translatePostgresExecutionToApiExecution translates a Postgres execution t updatedAt: executionRecord.updated_at.getTime(), }; - const result = await translatePostgresExecutionToApiExecution( + const translation = await translatePostgresExecutionToApiExecution( executionRecord, {}, fakeCollectionPgModel, @@ -104,7 +114,20 @@ test('translatePostgresExecutionToApiExecution translates a Postgres execution t ); t.deepEqual( - result, + translation, + expectedApiExecution + ); + + const duckDbTranslation = await translatePostgresExecutionToApiExecution( + duckDbExecutionRecord, + {}, + fakeCollectionPgModel, + fakeAsyncOperationPgModel, + fakeExecutionPgModel + ); + + t.deepEqual( + duckDbTranslation, expectedApiExecution ); }); diff --git a/packages/db/tests/translate/test-granules.js b/packages/db/tests/translate/test-granules.js index 9b3f226fa326..035c5a3cd2f4 100644 --- a/packages/db/tests/translate/test-granules.js +++ b/packages/db/tests/translate/test-granules.js @@ -153,7 +153,7 @@ test.beforeEach(async (t) => { `file0-${cryptoRandomString({ length: 10 })}`, `file1-${cryptoRandomString({ length: 10 })}`, ].sort(); - const files = [ + t.context.files = [ fakeFileRecordFactory({ bucket: 'cumulus-test-sandbox-private', checksum_type: 'md5', @@ -181,7 +181,9 @@ test.beforeEach(async (t) => { updated_at: updatedAt, }), ]; - await Promise.all(files.map((file) => t.context.filePgModel.create(t.context.knex, file))); + await Promise.all( + t.context.files.map((file) => t.context.filePgModel.create(t.context.knex, file)) + ); }); test('translatePostgresGranuleToApiGranule converts Postgres granule to API granule', async (t) => { @@ -196,6 +198,21 @@ test('translatePostgresGranuleToApiGranule converts Postgres granule to API gran executions, } = t.context; + const duckDbPostgresGranule = { + ...postgresGranule, + created_at: postgresGranule.created_at.toISOString(), + updated_at: postgresGranule.updated_at.toISOString(), + timestamp: postgresGranule.timestamp.toISOString(), + beginning_date_time: postgresGranule.beginning_date_time.toISOString(), + ending_date_time: postgresGranule.ending_date_time.toISOString(), + last_update_date_time: postgresGranule.last_update_date_time.toISOString(), + processing_end_date_time: postgresGranule.processing_end_date_time.toISOString(), + processing_start_date_time: postgresGranule.processing_start_date_time.toISOString(), + production_date_time: postgresGranule.production_date_time.toISOString(), + error: JSON.stringify(postgresGranule.error), + query_fields: JSON.stringify(postgresGranule.query_fields), + }; + const expectedApiGranule = { archived: false, beginningDateTime: postgresGranule.beginning_date_time.toISOString(), @@ -244,7 +261,7 @@ test('translatePostgresGranuleToApiGranule converts Postgres granule to API gran ], }; - const result = await translatePostgresGranuleToApiGranule({ + const translation = await translatePostgresGranuleToApiGranule({ granulePgRecord: postgresGranule, knexOrTransaction: knex, collectionPgModel, @@ -252,12 +269,33 @@ test('translatePostgresGranuleToApiGranule converts Postgres granule to API gran providerPgModel, filePgModel, }); - result.files.sort((a, b) => (a.fileName > b.fileName ? 1 : -1)); + translation.files.sort((a, b) => (a.fileName > b.fileName ? 1 : -1)); t.deepEqual( { - ...result, - files: orderBy(result.files, ['bucket', 'key']), + ...translation, + files: orderBy(translation.files, ['bucket', 'key']), + }, + { + ...expectedApiGranule, + files: orderBy(expectedApiGranule.files, ['bucket', 'key']), + } + ); + + const duckDbTranslation = await translatePostgresGranuleToApiGranule({ + granulePgRecord: duckDbPostgresGranule, + knexOrTransaction: knex, + collectionPgModel, + pdrPgModel, + providerPgModel, + filePgModel, + }); + duckDbTranslation.files.sort((a, b) => (a.fileName > b.fileName ? 1 : -1)); + + t.deepEqual( + { + ...duckDbTranslation, + files: orderBy(duckDbTranslation.files, ['bucket', 'key']), }, { ...expectedApiGranule, diff --git a/packages/db/tests/translate/test-pdr.js b/packages/db/tests/translate/test-pdr.js index 9c0a2aa3ccac..d65920bb2f90 100644 --- a/packages/db/tests/translate/test-pdr.js +++ b/packages/db/tests/translate/test-pdr.js @@ -46,6 +46,14 @@ test('translatePostgresPdrToApiPdr translates postgres PDR record to API PDR rec updated_at: timestamp, }; + const duckDbPdrRecord = { + ...postgresPdr, + created_at: postgresPdr.created_at.toISOString(), + updated_at: postgresPdr.updated_at.toISOString(), + timestamp: postgresPdr.timestamp.toISOString(), + stats: JSON.stringify(postgresPdr.stats), + }; + const expectedPdr = { pdrName: postgresPdr.name, provider: fakeProvider.name, @@ -76,6 +84,19 @@ test('translatePostgresPdrToApiPdr translates postgres PDR record to API PDR rec translatedPdrRecord, expectedPdr ); + + const duckDbTranslation = await translatePostgresPdrToApiPdr( + duckDbPdrRecord, + {}, + fakeCollectionPgModel, + fakeProviderPgModel, + fakeExecutionPgModel + ); + + t.deepEqual( + duckDbTranslation, + expectedPdr + ); }); test('translatePostgresPdrToApiPdr handles optional fields', async (t) => { diff --git a/packages/db/tests/translate/test-providers.js b/packages/db/tests/translate/test-providers.js index a2ab4fba3cf5..dfe2f68bb2b4 100644 --- a/packages/db/tests/translate/test-providers.js +++ b/packages/db/tests/translate/test-providers.js @@ -29,6 +29,12 @@ test('translatePostgresProviderToApiProvider translates the expected API record' allowed_redirects: allowedRedirects, }; + const duckDbPostgresProvider = { + ...postgresProviderObject, + created_at: postgresProviderObject.created_at.toISOString(), + updated_at: postgresProviderObject.updated_at.toISOString(), + }; + const expected = { certificateUri: 'fakeUri', cmKeyId: 'fakecmId', @@ -47,8 +53,11 @@ test('translatePostgresProviderToApiProvider translates the expected API record' allowedRedirects, }; - const result = translatePostgresProviderToApiProvider(postgresProviderObject); - t.deepEqual(result, expected); + const translation = translatePostgresProviderToApiProvider(postgresProviderObject); + t.deepEqual(translation, expected); + + const duckDbTranslation = translatePostgresProviderToApiProvider(duckDbPostgresProvider); + t.deepEqual(duckDbTranslation, expected); }); test('translatePostgresProviderToApiProvider does not return encrypted key if username and password are not present', (t) => { diff --git a/packages/db/tests/translate/test-reconciliation-reports.js b/packages/db/tests/translate/test-reconciliation-reports.js index 2314c78f7009..2a6d851107c7 100644 --- a/packages/db/tests/translate/test-reconciliation-reports.js +++ b/packages/db/tests/translate/test-reconciliation-reports.js @@ -18,6 +18,12 @@ test('translatePostgresReconReportToApiReconReport translates a Postgres Reconci updated_at: updatedTime, }; + const duckDbReconReport = { + ...pgReconReport, + created_at: pgReconReport.created_at.toISOString(), + updated_at: pgReconReport.updated_at.toISOString(), + }; + const expectedApiReconReport = { ...pick(pgReconReport, ['name', 'type', 'status', 'location']), // no error b/c null or undefined should be removed @@ -26,8 +32,10 @@ test('translatePostgresReconReportToApiReconReport translates a Postgres Reconci }; const translatedReport = translatePostgresReconReportToApiReconReport(pgReconReport); + t.deepEqual(translatedReport, expectedApiReconReport); - t.deepEqual(expectedApiReconReport, translatedReport); + const duckDbTranslatedReport = translatePostgresReconReportToApiReconReport(duckDbReconReport); + t.deepEqual(duckDbTranslatedReport, expectedApiReconReport); }); test('translatePostgresReconReportToApiReconReport translates Postgres Reconciliation Report with an error to an API Reconciliation Report', (t) => { @@ -47,6 +55,13 @@ test('translatePostgresReconReportToApiReconReport translates Postgres Reconcili updated_at: updatedTime, }; + const duckDbReconReport = { + ...pgReconReport, + created_at: pgReconReport.created_at.toISOString(), + updated_at: pgReconReport.updated_at.toISOString(), + error: JSON.stringify(pgReconReport.error), + }; + const expectedApiReconReport = { ...pick(pgReconReport, ['name', 'type', 'status', 'location', 'error']), createdAt: createdTime.getTime(), @@ -54,6 +69,8 @@ test('translatePostgresReconReportToApiReconReport translates Postgres Reconcili }; const translatedReport = translatePostgresReconReportToApiReconReport(pgReconReport); + t.deepEqual(translatedReport, expectedApiReconReport); - t.deepEqual(expectedApiReconReport, translatedReport); + const duckDbTranslatedReport = translatePostgresReconReportToApiReconReport(duckDbReconReport); + t.deepEqual(duckDbTranslatedReport, expectedApiReconReport); }); diff --git a/packages/db/tests/translate/test-rules.js b/packages/db/tests/translate/test-rules.js index b7ddd5acec37..1780164a6621 100644 --- a/packages/db/tests/translate/test-rules.js +++ b/packages/db/tests/translate/test-rules.js @@ -178,6 +178,15 @@ test('translatePostgresRuleToApiRule converts Postgres rule record to API rule', updated_at: new Date(), }; + const duckDbRuleRecord = { + ...pgRecord, + created_at: pgRecord.created_at.toISOString(), + updated_at: pgRecord.updated_at.toISOString(), + payload: JSON.stringify(pgRecord.payload), + meta: JSON.stringify(pgRecord.meta), + tags: JSON.stringify(pgRecord.tags), + }; + const fakeDbClient = {}; const fakeCollection = { name: 'abc', version: '123' }; const fakeCollectionPgModel = { @@ -218,6 +227,16 @@ test('translatePostgresRuleToApiRule converts Postgres rule record to API rule', ), expectedRule ); + + t.deepEqual( + await translatePostgresRuleToApiRule( + duckDbRuleRecord, + fakeDbClient, + fakeCollectionPgModel, + fakeProviderPgModel + ), + expectedRule + ); }); test('translatePostgresRuleToApiRule handles optional fields', async (t) => { From b0002524a85458f73f9852c601fea49793f59272 Mon Sep 17 00:00:00 2001 From: Tim Clark <156814997+tclark-innovim@users.noreply.github.com> Date: Mon, 23 Feb 2026 11:57:18 -0500 Subject: [PATCH 13/26] Release 21.3.1 merge to master (#4260) * increment version to 21.3.0 * increment async-operation to 55 * update CL * update contributors * update package api docs * add website documentation * Release 21.3.0 (#4227) * increment version to 21.3.0 * increment async-operation to 55 * update CL * update contributors * update package api docs * add website documentation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Tim Clark Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Release 21.3.0 (#4233) * Update lerna to v8, add tar override shim (#4230) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Tim Clark Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * fix CL 4458 * add tar_shim to npm publish * fix compare in CL * update async_operation_image_version to 55 * update async_operation_image_version to 55 * update ecs async-operation Dockerfile to node 22 * increment version in website package.json * Release 21.3.1 (#4255) * added states:StartExecution action to the -steprole IAM role * Overriding vulnerable dependency in @aws-client/xml-builder (cherry picked from commit eeefc18f6232be4bee4ea6a883a1519b602a1874) * Moved fast-xml-parser override to the root package.json (cherry picked from commit d32ab155088f37d74a9c26ddf886a36baff6e618) * CUMULUS-4514 cherry pick * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * version bump to 21.3.1 --------- Co-authored-by: Tim Clark Co-authored-by: mikedorfman Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * update changelog for v21.3.1 * update CL --------- Co-authored-by: Tim Clark Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: mikedorfman --- CHANGELOG.md | 17 ++++-- example/lambdas/asyncOperations/package.json | 2 +- .../ftpPopulateTestLambda/package.json | 14 ++--- example/lambdas/lzardsClientTest/package.json | 6 +-- .../lambdas/python-processing/package.json | 2 +- .../python-reference-activity/package.json | 2 +- .../python-reference-task/package.json | 2 +- example/lambdas/s3AccessTest/package.json | 2 +- example/lambdas/snsS3Test/package.json | 2 +- example/lambdas/versionUpTest/package.json | 2 +- example/package.json | 52 +++++++++---------- example/scripts/generate_ingest/package.json | 6 +-- example/scripts/lib/package.json | 2 +- lambdas/db-migration/package.json | 4 +- .../db-provision-user-database/package.json | 6 +-- lambdas/dla-migration/package.json | 12 ++--- .../package.json | 14 ++--- lambdas/sqs-message-remover/package.json | 10 ++-- lerna.json | 2 +- package.json | 5 +- packages/api-client/package.json | 8 +-- packages/api/ecs/async-operation/package.json | 8 +-- packages/api/package.json | 44 ++++++++-------- packages/async-operations/package.json | 16 +++--- packages/aws-client/package.json | 12 ++--- packages/checksum/package.json | 2 +- packages/cmr-client/package.json | 10 ++-- packages/cmrjs/package.json | 20 +++---- packages/collection-config-store/package.json | 8 +-- packages/common/package.json | 10 ++-- packages/db/package.json | 14 ++--- packages/distribution-utils/package.json | 8 +-- packages/errors/package.json | 2 +- packages/es-client/package.json | 14 ++--- packages/ingest/package.json | 24 ++++----- packages/integration-tests/package.json | 22 ++++---- packages/launchpad-auth/package.json | 6 +-- packages/logger/package.json | 2 +- packages/lzards-api-client/package.json | 12 ++--- packages/message/package.json | 12 ++--- packages/oauth-client/package.json | 2 +- packages/object-store/package.json | 4 +- packages/pvl/package.json | 2 +- packages/s3-credentials-endpoint/package.json | 14 ++--- packages/schemas/package.json | 2 +- packages/sftp-client/package.json | 10 ++-- packages/tea-map-cache/package.json | 6 +-- packages/test-data/package.json | 2 +- packages/tf-inventory/package.json | 6 +-- packages/types/package.json | 2 +- scripts/generate_records/package.json | 14 ++--- scripts/unit-error-analysis/package.json | 4 +- tasks/add-missing-file-checksums/package.json | 8 +-- tasks/add-unique-granuleID/package.json | 12 ++--- .../change-granule-collection-pg/package.json | 22 ++++---- .../change-granule-collection-s3/package.json | 22 ++++---- tasks/discover-granules/package.json | 12 ++--- tasks/discover-pdrs/package.json | 10 ++-- tasks/files-to-granules/package.json | 10 ++-- tasks/hello-world/package.json | 6 +-- tasks/hyrax-metadata-updates/package.json | 14 ++--- tasks/lzards-backup/package.json | 24 ++++----- tasks/move-granules/package.json | 20 +++---- .../orca-copy-to-archive-adapter/package.json | 10 ++-- tasks/orca-recovery-adapter/package.json | 12 ++--- tasks/parse-pdr/package.json | 20 +++---- tasks/pdr-status-check/package.json | 8 +-- tasks/post-to-cmr/package.json | 16 +++--- tasks/queue-granules/package.json | 12 ++--- tasks/queue-pdrs/package.json | 10 ++-- tasks/queue-workflow/package.json | 10 ++-- tasks/send-pan/package.json | 12 ++--- tasks/sf-sqs-report/package.json | 6 +-- tasks/sync-granule/package.json | 22 ++++---- tasks/test-processing/package.json | 6 +-- .../package.json | 10 ++-- .../package.json | 14 ++--- tf-modules/ingest/package.json | 2 +- .../cumulus-test-cleanup/package.json | 2 +- tf-modules/s3-replicator/package.json | 6 +-- website/package.json | 2 +- 81 files changed, 420 insertions(+), 406 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a358917e575a..b29c4449062f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added python code for CnmResponse task adapted from https://github.com/podaac/cumulus-cnm-response-task - **CUMULUS-4395** - Added supporting Terraform for the CnmResponse task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus. -- **CUMULUS-4498** - - Added `states:StartExecution` action to the `-steprole` IAM role. - **CUMULUS-4517** - Added the `@cumulus/db/s3search` module to enable Cumulus record search via S3-backed tables. The S3Search subclasses inherit from search/BaseSearch, allowing them to reuse existing query @@ -74,6 +72,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Updated sftp-client to explicitly tear down stream in sftp-client/syncFromS3 - Updated sftp-client to warn/log on `No response from server` errors in `end` method +## [v21.3.1] 2026-02-16 + +### Added + +- **CUMULUS-4498** + - Added `states:StartExecution` action to the `-steprole` IAM role. + +### Changed + +- **CUMULUS-4514** + - Pinned fast-xml-parser at 5.3.4 for @aws-sdk/xml-builder due to a security vulnerability. + ## [v21.3.0] 2026-01-26 ### Migration Notes @@ -9502,7 +9512,8 @@ Note: There was an issue publishing 1.12.0. Upgrade to 1.12.1. ## [v1.0.0] - 2018-02-23 -[Unreleased]: https://github.com/nasa/cumulus/compare/v21.3.0...HEAD +[Unreleased]: https://github.com/nasa/cumulus/compare/v21.3.1...HEAD +[v21.3.0]: https://github.com/nasa/cumulus/compare/v21.3.0...v21.3.1 [v21.3.0]: https://github.com/nasa/cumulus/compare/v21.2.0...v21.3.0 [v21.2.0]: https://github.com/nasa/cumulus/compare/v21.0.1...v21.2.0 [v21.0.1]: https://github.com/nasa/cumulus/compare/v21.0.0...v21.0.1 diff --git a/example/lambdas/asyncOperations/package.json b/example/lambdas/asyncOperations/package.json index ef1c8215da4d..c3d55445a66f 100644 --- a/example/lambdas/asyncOperations/package.json +++ b/example/lambdas/asyncOperations/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-async-operations", - "version": "21.3.0", + "version": "21.3.1", "description": "AsyncOperations Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/ftpPopulateTestLambda/package.json b/example/lambdas/ftpPopulateTestLambda/package.json index f5ed2e4b2671..3c109c75a4fa 100644 --- a/example/lambdas/ftpPopulateTestLambda/package.json +++ b/example/lambdas/ftpPopulateTestLambda/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ftp-populate-test-lambda", - "version": "21.3.0", + "version": "21.3.1", "description": "FTP Population Utility Lambda", "main": "index.js", "private": true, @@ -19,12 +19,12 @@ "access": "private" }, "dependencies": { - "@cumulus/api": "21.3.0", - "@cumulus/api-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/integration-tests": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/test-data": "21.3.0", + "@cumulus/api": "21.3.1", + "@cumulus/api-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/integration-tests": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/test-data": "21.3.1", "fs-extra": "^9.0.0", "jsftp": "https://github.com/jkovarik/jsftp.git#add_288", "lodash": "^4.17.20" diff --git a/example/lambdas/lzardsClientTest/package.json b/example/lambdas/lzardsClientTest/package.json index 825bdfba3ae0..7748a7a3fba3 100644 --- a/example/lambdas/lzardsClientTest/package.json +++ b/example/lambdas/lzardsClientTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-lzards-api-lambda", - "version": "21.3.0", + "version": "21.3.1", "description": "LZARDS API Client Test Lambda", "private": true, "engines": { @@ -20,7 +20,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/logger": "21.3.0", - "@cumulus/lzards-api-client": "21.3.0" + "@cumulus/logger": "21.3.1", + "@cumulus/lzards-api-client": "21.3.1" } } diff --git a/example/lambdas/python-processing/package.json b/example/lambdas/python-processing/package.json index fe7b43a5b84b..a7a4f349bde7 100644 --- a/example/lambdas/python-processing/package.json +++ b/example/lambdas/python-processing/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-process-activity", "private": true, - "version": "21.3.0", + "version": "21.3.1", "description": "Python reference activity", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity", "repository": { diff --git a/example/lambdas/python-reference-activity/package.json b/example/lambdas/python-reference-activity/package.json index 1e976847a477..469e499b4cca 100644 --- a/example/lambdas/python-reference-activity/package.json +++ b/example/lambdas/python-reference-activity/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-reference-activity", "private": true, - "version": "21.3.0", + "version": "21.3.1", "description": "Python reference activity", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity", "repository": { diff --git a/example/lambdas/python-reference-task/package.json b/example/lambdas/python-reference-task/package.json index 62bcd1cf0760..ac8f6b52ef62 100644 --- a/example/lambdas/python-reference-task/package.json +++ b/example/lambdas/python-reference-task/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-reference-task", "private": true, - "version": "21.3.0", + "version": "21.3.1", "description": "Python reference task", "main": "index.js", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-task", diff --git a/example/lambdas/s3AccessTest/package.json b/example/lambdas/s3AccessTest/package.json index 2828eda98d28..9cebba4968d9 100644 --- a/example/lambdas/s3AccessTest/package.json +++ b/example/lambdas/s3AccessTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-s3-access", - "version": "21.3.0", + "version": "21.3.1", "description": "S3 Access Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/snsS3Test/package.json b/example/lambdas/snsS3Test/package.json index 0a8364af91db..70bab053db02 100644 --- a/example/lambdas/snsS3Test/package.json +++ b/example/lambdas/snsS3Test/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-sns-s3", - "version": "21.3.0", + "version": "21.3.1", "description": "SNS to S3 Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/versionUpTest/package.json b/example/lambdas/versionUpTest/package.json index b22f432d0e27..2d68c1a97d30 100644 --- a/example/lambdas/versionUpTest/package.json +++ b/example/lambdas/versionUpTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-version-up", - "version": "21.3.0", + "version": "21.3.1", "description": "Version Up Test Lambda", "main": "index.js", "private": true, diff --git a/example/package.json b/example/package.json index 937c2acec7f2..9ec40b268f9b 100644 --- a/example/package.json +++ b/example/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cumulus-integration-tests", - "version": "21.3.0", + "version": "21.3.1", "description": "Cumulus Integration Test Deployment", "private": true, "main": "index.js", @@ -49,32 +49,32 @@ "@aws-sdk/client-lambda": "^3.993.0", "@aws-sdk/client-sns": "^3.993.0", "@aws-sdk/client-sts": "^3.993.0", - "@cumulus/api": "21.3.0", - "@cumulus/api-client": "21.3.0", - "@cumulus/async-operations": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/checksum": "21.3.0", - "@cumulus/cmr-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/discover-granules": "21.3.0", - "@cumulus/discover-pdrs": "21.3.0", - "@cumulus/files-to-granules": "21.3.0", - "@cumulus/hello-world": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/integration-tests": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/move-granules": "21.3.0", - "@cumulus/parse-pdr": "21.3.0", - "@cumulus/pdr-status-check": "21.3.0", - "@cumulus/post-to-cmr": "21.3.0", - "@cumulus/queue-granules": "21.3.0", - "@cumulus/queue-pdrs": "21.3.0", - "@cumulus/sf-sqs-report": "21.3.0", - "@cumulus/sync-granule": "21.3.0", - "@cumulus/test-processing": "21.3.0" + "@cumulus/api": "21.3.1", + "@cumulus/api-client": "21.3.1", + "@cumulus/async-operations": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/checksum": "21.3.1", + "@cumulus/cmr-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/discover-granules": "21.3.1", + "@cumulus/discover-pdrs": "21.3.1", + "@cumulus/files-to-granules": "21.3.1", + "@cumulus/hello-world": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/integration-tests": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/move-granules": "21.3.1", + "@cumulus/parse-pdr": "21.3.1", + "@cumulus/pdr-status-check": "21.3.1", + "@cumulus/post-to-cmr": "21.3.1", + "@cumulus/queue-granules": "21.3.1", + "@cumulus/queue-pdrs": "21.3.1", + "@cumulus/sf-sqs-report": "21.3.1", + "@cumulus/sync-granule": "21.3.1", + "@cumulus/test-processing": "21.3.1" }, "devDependencies": { - "@cumulus/test-data": "21.3.0" + "@cumulus/test-data": "21.3.1" } } diff --git a/example/scripts/generate_ingest/package.json b/example/scripts/generate_ingest/package.json index 2c90fdb87283..54859265f354 100644 --- a/example/scripts/generate_ingest/package.json +++ b/example/scripts/generate_ingest/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate_ingest", "private": true, - "version": "21.3.0", + "version": "21.3.1", "description": "Script to generate test data for scaled ingest", "keywords": [ "GIBS", @@ -22,8 +22,8 @@ "directory": "packages/types" }, "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0" + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1" }, "author": "Cumulus Authors", "license": "Apache-2.0" diff --git a/example/scripts/lib/package.json b/example/scripts/lib/package.json index 7340618b32f7..8116c836c0ca 100644 --- a/example/scripts/lib/package.json +++ b/example/scripts/lib/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/example-lib", "private": true, - "version": "21.3.0", + "version": "21.3.1", "description": "example project libs", "homepage": "https://github.com/nasa/cumulus/tree/master/example/scripts/lib", "engines": { diff --git a/lambdas/db-migration/package.json b/lambdas/db-migration/package.json index d815bf9c7d5b..b9206475eeb3 100644 --- a/lambdas/db-migration/package.json +++ b/lambdas/db-migration/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db-migration-lambda", - "version": "21.3.0", + "version": "21.3.1", "description": "A Lambda function used for deploying DB migrations", "license": "Apache-2.0", "engines": { @@ -20,7 +20,7 @@ "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles" }, "dependencies": { - "@cumulus/db": "21.3.0", + "@cumulus/db": "21.3.1", "knex": "2.4.1", "pg": "~8.13" } diff --git a/lambdas/db-provision-user-database/package.json b/lambdas/db-provision-user-database/package.json index b618da6e87a1..e871123e6923 100644 --- a/lambdas/db-provision-user-database/package.json +++ b/lambdas/db-provision-user-database/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db-provision-user-database-lambda", - "version": "21.3.0", + "version": "21.3.1", "description": "A Lambda function used for provisioning user databases", "engines": { "node": ">=22.21.1" @@ -29,8 +29,8 @@ }, "dependencies": { "@aws-sdk/client-secrets-manager": "^3.993.0", - "@cumulus/common": "21.3.0", - "@cumulus/db": "21.3.0", + "@cumulus/common": "21.3.1", + "@cumulus/db": "21.3.1", "knex": "2.4.1", "lodash": "^4.17.21", "pg": "~8.13" diff --git a/lambdas/dla-migration/package.json b/lambdas/dla-migration/package.json index d6a0f70c0090..89e0b45308d6 100644 --- a/lambdas/dla-migration/package.json +++ b/lambdas/dla-migration/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/dla-migration-lambda", - "version": "21.3.0", + "version": "21.3.1", "description": "A Lambda function used for DLA migrations", "license": "Apache-2.0", "engines": { @@ -24,11 +24,11 @@ "webpack": "../../node_modules/.bin/webpack" }, "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/types": "21.3.1", "lodash": "^4.17.21", "moment": "^2.30.1", "p-map": "^4.0.0" diff --git a/lambdas/migration-helper-async-operation/package.json b/lambdas/migration-helper-async-operation/package.json index 97469475cbad..5f51e03c2223 100644 --- a/lambdas/migration-helper-async-operation/package.json +++ b/lambdas/migration-helper-async-operation/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/migration-helper-async-operation", - "version": "21.3.0", + "version": "21.3.1", "description": "Lambda function helps with various migrations", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -31,12 +31,12 @@ "failFast": true }, "dependencies": { - "@cumulus/api": "21.3.0", - "@cumulus/async-operations": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/api": "21.3.1", + "@cumulus/async-operations": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/types": "21.3.1", "lodash": "^4.17.21" } } diff --git a/lambdas/sqs-message-remover/package.json b/lambdas/sqs-message-remover/package.json index 306ad41864e8..9263c0d8d035 100644 --- a/lambdas/sqs-message-remover/package.json +++ b/lambdas/sqs-message-remover/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sqs-message-remover-lambda", - "version": "21.3.0", + "version": "21.3.1", "description": "Remove processed messages from SQS queues", "main": "src/index.js", "private": true, @@ -39,10 +39,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/logger": "21.3.1", "lodash": "^4.17.21" } } diff --git a/lerna.json b/lerna.json index 594fee299298..2a36166236c9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "3.20.2", - "version": "21.3.0", + "version": "21.3.1", "packages": [ "example", "example/lambdas/*", diff --git a/package.json b/package.json index 27680e6ebba5..bb92ee6642ec 100644 --- a/package.json +++ b/package.json @@ -190,7 +190,10 @@ "request": { "qs": "^6.14.1" }, - "tar": "^7.5.3" + "tar": "^7.5.3", + "@aws-sdk/xml-builder": { + "fast-xml-parser": "5.3.4" + } }, "private": true } diff --git a/packages/api-client/package.json b/packages/api-client/package.json index b32fc623066f..64f0f91c519a 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api-client", - "version": "21.3.0", + "version": "21.3.1", "description": "API client for working with the Cumulus archive API", "keywords": [ "GIBS", @@ -42,11 +42,11 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-lambda": "^3.993.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/logger": "21.3.1", "p-retry": "^2.0.0" }, "devDependencies": { - "@cumulus/types": "21.3.0" + "@cumulus/types": "21.3.1" } } diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index 77271723a743..8950c1f33c9a 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ecs-async-operation", - "version": "21.3.0", + "version": "21.3.1", "description": "The docker image for running async operations", "keywords": [ "NASA", @@ -23,9 +23,9 @@ }, "dependencies": { "@aws-sdk/client-lambda": "^3.993.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/db": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/db": "21.3.1", + "@cumulus/logger": "21.3.1", "crypto-random-string": "^3.2.0", "got": "^11.8.5", "lodash": "^4.17.21", diff --git a/packages/api/package.json b/packages/api/package.json index 586fdef26dc4..f85cf0c8fcb0 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api", - "version": "21.3.0", + "version": "21.3.1", "description": "Lambda functions for handling all daac's API operations", "main": "index.js", "engines": { @@ -55,26 +55,26 @@ "@aws-sdk/client-sfn": "^3.993.0", "@aws-sdk/client-sns": "^3.993.0", "@aws-sdk/client-sqs": "^3.993.0", - "@cumulus/api-client": "21.3.0", - "@cumulus/async-operations": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmr-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/collection-config-store": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/db": "21.3.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/es-client": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/launchpad-auth": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/oauth-client": "21.3.0", - "@cumulus/object-store": "21.3.0", - "@cumulus/pvl": "21.3.0", - "@cumulus/sftp-client": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/async-operations": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmr-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/collection-config-store": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/db": "21.3.1", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/es-client": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/launchpad-auth": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/oauth-client": "21.3.1", + "@cumulus/object-store": "21.3.1", + "@cumulus/pvl": "21.3.1", + "@cumulus/sftp-client": "21.3.1", + "@cumulus/types": "21.3.1", "@hapi/boom": "^10.0.0", "@mapbox/dyno": "^1.4.2", "aggregate-error": "^3.1.0", @@ -127,7 +127,7 @@ } }, "devDependencies": { - "@cumulus/test-data": "21.3.0", + "@cumulus/test-data": "21.3.1", "aws-sdk-client-mock": "^3.0.1", "proxyquire": "^2.1.3" } diff --git a/packages/async-operations/package.json b/packages/async-operations/package.json index cf081ede8c66..08addba03f03 100644 --- a/packages/async-operations/package.json +++ b/packages/async-operations/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/async-operations", - "version": "21.3.0", + "version": "21.3.1", "description": "Cumulus Core internal async operations module", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -32,17 +32,17 @@ "dependencies": { "@aws-sdk/client-ecs": "^3.993.0", "@aws-sdk/client-lambda": "^3.993.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/db": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/es-client": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/db": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/es-client": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/types": "21.3.1", "knex": "2.4.1", "uuid": "8.3.2" }, "devDependencies": { - "@cumulus/common": "21.3.0", + "@cumulus/common": "21.3.1", "@types/aws-sdk": "2.7.0", "@types/uuid": "^8.0.0" } diff --git a/packages/aws-client/package.json b/packages/aws-client/package.json index 6143604f5f25..2f19adf97927 100644 --- a/packages/aws-client/package.json +++ b/packages/aws-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/aws-client", - "version": "21.3.0", + "version": "21.3.1", "description": "Utilities for working with AWS", "keywords": [ "GIBS", @@ -69,10 +69,10 @@ "@aws-sdk/s3-request-presigner": "^3.993.0", "@aws-sdk/signature-v4-crt": "^3.993.0", "@aws-sdk/types": "^3.609.0", - "@cumulus/checksum": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/checksum": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/types": "21.3.1", "lodash": "~4.17.21", "mem": "^8.0.2", "p-map": "^1.2.0", @@ -83,7 +83,7 @@ "uuid": "^8.2.0" }, "devDependencies": { - "@cumulus/test-data": "21.3.0", + "@cumulus/test-data": "21.3.1", "@types/uuid": "^8.0.0" } } diff --git a/packages/checksum/package.json b/packages/checksum/package.json index d24b72d863a8..d8361ac03415 100644 --- a/packages/checksum/package.json +++ b/packages/checksum/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/checksum", - "version": "21.3.0", + "version": "21.3.1", "description": "Cumulus checksum utilities", "engines": { "node": ">=22.21.1" diff --git a/packages/cmr-client/package.json b/packages/cmr-client/package.json index 6d9ca4738bee..240cfaed4ff5 100644 --- a/packages/cmr-client/package.json +++ b/packages/cmr-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmr-client", - "version": "21.3.0", + "version": "21.3.1", "description": "A Node.js client to NASA's Common Metadata Repository (CMR) API.", "engines": { "node": ">=22.21.1" @@ -36,10 +36,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", "got": "^11.8.5", "jsonwebtoken": "^9.0.0", "lodash": "^4.17.21", diff --git a/packages/cmrjs/package.json b/packages/cmrjs/package.json index f8f59deabe34..25e3a38f9987 100644 --- a/packages/cmrjs/package.json +++ b/packages/cmrjs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmrjs", - "version": "21.3.0", + "version": "21.3.1", "description": "A node SDK for CMR", "engines": { "node": ">=22.21.1" @@ -36,14 +36,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmr-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/launchpad-auth": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmr-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/launchpad-auth": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", "got": "^11.8.1", "js2xmlparser": "^5.0.0", "lodash": "^4.17.21", @@ -52,6 +52,6 @@ "xml2js": "0.5.0" }, "devDependencies": { - "@cumulus/types": "21.3.0" + "@cumulus/types": "21.3.1" } } diff --git a/packages/collection-config-store/package.json b/packages/collection-config-store/package.json index 9de441cdaf77..ed8c9ac39da3 100644 --- a/packages/collection-config-store/package.json +++ b/packages/collection-config-store/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/collection-config-store", - "version": "21.3.0", + "version": "21.3.1", "description": "Utility for persisting collection configuration to S3 and retrieving it", "keywords": [ "CUMULUS", @@ -35,8 +35,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/message": "21.3.0" + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/message": "21.3.1" } } diff --git a/packages/common/package.json b/packages/common/package.json index c5226dd48b18..23926ac0c394 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/common", - "version": "21.3.0", + "version": "21.3.1", "description": "Common utilities used across tasks", "keywords": [ "GIBS", @@ -46,10 +46,10 @@ "dependencies": { "@aws-sdk/client-s3": "^3.993.0", "@aws-sdk/signature-v4-crt": "^3.993.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "ajv": "^8.18.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "ajv": "^6.12.3", "follow-redirects": "^1.2.4", "fs-extra": "^5.0.0", "got": "^14.2.1", diff --git a/packages/db/package.json b/packages/db/package.json index 076ecc19f0a1..9bb0792725e0 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db", - "version": "21.3.0", + "version": "21.3.1", "description": "Utilities for working with the Cumulus DB", "license": "Apache-2.0", "main": "./dist/index.js", @@ -33,12 +33,12 @@ }, "dependencies": { "@aws-sdk/client-secrets-manager": "^3.993.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/types": "21.3.1", "@duckdb/node-api": "^1.4.4-r.1", "crypto-random-string": "^3.2.0", "is-valid-hostname": "1.0.2", diff --git a/packages/distribution-utils/package.json b/packages/distribution-utils/package.json index 5ad26664126f..0c3409d5f3ad 100644 --- a/packages/distribution-utils/package.json +++ b/packages/distribution-utils/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/distribution-utils", - "version": "21.3.0", + "version": "21.3.1", "description": "Cumulus Distribution utilities", "keywords": [ "CUMULUS" @@ -35,9 +35,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1", "url-join": "^1.1.0" }, "devDependencies": { diff --git a/packages/errors/package.json b/packages/errors/package.json index a9850f6769d0..e26ccef2a309 100644 --- a/packages/errors/package.json +++ b/packages/errors/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/errors", - "version": "21.3.0", + "version": "21.3.1", "description": "Provides error classes for Cumulus", "keywords": [ "GIBS", diff --git a/packages/es-client/package.json b/packages/es-client/package.json index 9e535d01de18..99e919031a7c 100644 --- a/packages/es-client/package.json +++ b/packages/es-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/es-client", - "version": "21.3.0", + "version": "21.3.1", "description": "Utilities for working with Elasticsearch", "keywords": [ "CUMULUS", @@ -34,10 +34,10 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/credential-providers": "^3.993.0", - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", "@elastic/elasticsearch": "^5.6.20", "aws4": "^1.12.0", "lodash": "~4.17.21", @@ -45,8 +45,8 @@ "p-limit": "^1.2.0" }, "devDependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/test-data": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/test-data": "21.3.1", "p-each-series": "^2.1.0" } } diff --git a/packages/ingest/package.json b/packages/ingest/package.json index de1ef3bb6768..616a931a2397 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest", - "version": "21.3.0", + "version": "21.3.1", "description": "Ingest utilities", "engines": { "node": ">=22.21.1" @@ -41,13 +41,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/db": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/sftp-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/db": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/sftp-client": "21.3.1", "cksum": "^1.3.0", "encodeurl": "^1.0.2", "fs-extra": "^5.0.0", @@ -61,9 +61,9 @@ "tough-cookie": "~4.0.0" }, "devDependencies": { - "@cumulus/checksum": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/test-data": "21.3.0", - "@cumulus/types": "21.3.0" + "@cumulus/checksum": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/test-data": "21.3.1", + "@cumulus/types": "21.3.1" } } diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 0dbe3fe7ef30..e797f2db1904 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/integration-tests", - "version": "21.3.0", + "version": "21.3.1", "description": "Integration tests", "bin": { "cumulus-test": "./bin/cli.js" @@ -31,16 +31,16 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-lambda": "^3.993.0", - "@cumulus/api": "21.3.0", - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmr-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/launchpad-auth": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/oauth-client": "21.3.0", + "@cumulus/api": "21.3.1", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmr-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/launchpad-auth": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/oauth-client": "21.3.1", "base-64": "^0.1.0", "commander": "^2.15.0", "dotenv": "^8.2.0", diff --git a/packages/launchpad-auth/package.json b/packages/launchpad-auth/package.json index 59ccbba1574d..b979d3ff5c8d 100644 --- a/packages/launchpad-auth/package.json +++ b/packages/launchpad-auth/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/launchpad-auth", - "version": "21.3.0", + "version": "21.3.1", "description": "Utilities for authentication with Launchpad", "keywords": [ "CUMULUS", @@ -40,8 +40,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/logger": "21.3.1", "got": "^11.8.5", "lodash": "^4.17.21", "uuid": "^3.2.1" diff --git a/packages/logger/package.json b/packages/logger/package.json index af8fb55ec4ab..74c909301207 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/logger", - "version": "21.3.0", + "version": "21.3.1", "description": "A log library for use on Cumulus", "keywords": [ "GIBS", diff --git a/packages/lzards-api-client/package.json b/packages/lzards-api-client/package.json index ee77019ead66..e3e43ef4aee6 100644 --- a/packages/lzards-api-client/package.json +++ b/packages/lzards-api-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/lzards-api-client", - "version": "21.3.0", + "version": "21.3.1", "description": "A Node.js client to NASA's Level Zero and Repositories Data Store (LZARDS) API.", "engines": { "node": ">=22.21.1" @@ -35,11 +35,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/launchpad-auth": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/launchpad-auth": "21.3.1", + "@cumulus/logger": "21.3.1", "got": "^11.8.5", "lodash": "^4.17.21" } diff --git a/packages/message/package.json b/packages/message/package.json index 6650a2f46a10..ed7b3110625f 100644 --- a/packages/message/package.json +++ b/packages/message/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/message", - "version": "21.3.0", + "version": "21.3.1", "description": "Utilities for building and parsing Cumulus messages", "keywords": [ "GIBS", @@ -41,11 +41,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/types": "21.3.1", "jsonpath-plus": "^10.0.0", "lodash": "^4.17.21", "uuidv4": "^6.2.13" diff --git a/packages/oauth-client/package.json b/packages/oauth-client/package.json index 171d2b4e1235..badbff79a1c9 100644 --- a/packages/oauth-client/package.json +++ b/packages/oauth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/oauth-client", - "version": "21.3.0", + "version": "21.3.1", "description": "A generic auth client", "homepage": "https://github.com/nasa/cumulus/tree/master/packages/oauth-client#readme", "repository": { diff --git a/packages/object-store/package.json b/packages/object-store/package.json index 1ea1a09303d1..13209ed3f5a3 100644 --- a/packages/object-store/package.json +++ b/packages/object-store/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/object-store", - "version": "21.3.0", + "version": "21.3.1", "description": "Utilities for managing object stores", "keywords": [ "GIBS", @@ -42,6 +42,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0" + "@cumulus/aws-client": "21.3.1" } } diff --git a/packages/pvl/package.json b/packages/pvl/package.json index 6456a9ea225f..75940bb04b9b 100644 --- a/packages/pvl/package.json +++ b/packages/pvl/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pvl", - "version": "21.3.0", + "version": "21.3.1", "description": "Parse and serialize Parameter Value Language, a data markup language used by NASA", "main": "index.js", "engine": { diff --git a/packages/s3-credentials-endpoint/package.json b/packages/s3-credentials-endpoint/package.json index 00c18018bcd8..a24ce00977b5 100644 --- a/packages/s3-credentials-endpoint/package.json +++ b/packages/s3-credentials-endpoint/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/s3-credentials-endpoint", - "version": "21.3.0", + "version": "21.3.1", "description": "An API Gateway Lambda to return AWS credentials for fetching objects from S3", "license": "Apache-2.0", "engines": { @@ -20,12 +20,12 @@ "failFast": true }, "dependencies": { - "@cumulus/api": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/oauth-client": "21.3.0", + "@cumulus/api": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/oauth-client": "21.3.1", "aws-serverless-express": "^3.3.6", "body-parser": "^1.19.0", "cookie-parser": "^1.4.4", diff --git a/packages/schemas/package.json b/packages/schemas/package.json index fe4672e6399d..1d1b21202e9e 100644 --- a/packages/schemas/package.json +++ b/packages/schemas/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/schemas", - "version": "21.3.0", + "version": "21.3.1", "description": "Helpers for managing Cumulus task schemas", "homepage": "https://github.com/nasa/cumulus/tree/master/packages/schemas", "repository": { diff --git a/packages/sftp-client/package.json b/packages/sftp-client/package.json index c680dc108f4d..fa1c8e1fb862 100644 --- a/packages/sftp-client/package.json +++ b/packages/sftp-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sftp-client", - "version": "21.3.0", + "version": "21.3.1", "description": "A Promise-based SFTP client", "keywords": [ "GIBS", @@ -39,16 +39,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "lodash": "^4.17.21", "mime-types": "^2.1.27", "ssh2": "^1.16.0", "ssh2-sftp-client": "^11.0.0" }, "devDependencies": { - "@cumulus/checksum": "21.3.0", - "@cumulus/test-data": "21.3.0", + "@cumulus/checksum": "21.3.1", + "@cumulus/test-data": "21.3.1", "@types/ssh2-sftp-client": "^7.0.0" } } diff --git a/packages/tea-map-cache/package.json b/packages/tea-map-cache/package.json index c23a2789a8d2..4aaf040275fd 100644 --- a/packages/tea-map-cache/package.json +++ b/packages/tea-map-cache/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/tea-map-cache", - "version": "21.3.0", + "version": "21.3.1", "description": "Tea Bucket Map Cache Writer", "main": "index.js", "engines": { @@ -30,8 +30,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/logger": "21.3.1", "got": "^11.8.5", "p-retry": "^4.2.0" }, diff --git a/packages/test-data/package.json b/packages/test-data/package.json index 06ba0cc99108..3e18d9c25bd2 100644 --- a/packages/test-data/package.json +++ b/packages/test-data/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-data", - "version": "21.3.0", + "version": "21.3.1", "description": "Includes the test data for various packages", "keywords": [ "GIBS", diff --git a/packages/tf-inventory/package.json b/packages/tf-inventory/package.json index 7f77971b5d63..d32d08ec4508 100644 --- a/packages/tf-inventory/package.json +++ b/packages/tf-inventory/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/tf-inventory", - "version": "21.3.0", + "version": "21.3.1", "description": "Package to help keep track of what resources are managed by Terraform in the AWS account", "main": "index.js", "engines": { @@ -32,11 +32,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "commander": "^4.1.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "21.3.0" + "@cumulus/common": "21.3.1" } } diff --git a/packages/types/package.json b/packages/types/package.json index 5b3ca03bea6d..90b466098c26 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/types", - "version": "21.3.0", + "version": "21.3.1", "description": "TypeScript definitions for working with Cumulus data structures", "keywords": [ "GIBS", diff --git a/scripts/generate_records/package.json b/scripts/generate_records/package.json index d51ab9ceb040..5a13bb231a0c 100644 --- a/scripts/generate_records/package.json +++ b/scripts/generate_records/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/generate_records", - "version": "21.3.0", + "version": "21.3.1", "author": "Cumulus Authors", "description": "compile errors", "license": "Apache-2.0", @@ -22,12 +22,12 @@ "test:coverage": "../../node_modules/.bin/nyc npm test" }, "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/common": "21.3.0", - "@cumulus/db": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/integration-tests": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/common": "21.3.1", + "@cumulus/db": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/integration-tests": "21.3.1", + "@cumulus/logger": "21.3.1", "@types/minimist": "^1.2.5", "cli-progress": "^3.12.0", "crypto-js": "^4.2.0", diff --git a/scripts/unit-error-analysis/package.json b/scripts/unit-error-analysis/package.json index 0647523297d0..04d186b50833 100644 --- a/scripts/unit-error-analysis/package.json +++ b/scripts/unit-error-analysis/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/unit-error-analysis", - "version": "21.3.0", + "version": "21.3.1", "author": "Cumulus Authors", "description": "compile errors", "license": "Apache-2.0", @@ -26,7 +26,7 @@ "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles" }, "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "@types/minimist": "^1.2.5", "minimist": "^1.2.8", "moment": "^2.30.1", diff --git a/tasks/add-missing-file-checksums/package.json b/tasks/add-missing-file-checksums/package.json index e4ea41df2849..c4064cdc67aa 100644 --- a/tasks/add-missing-file-checksums/package.json +++ b/tasks/add-missing-file-checksums/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/add-missing-file-checksums", - "version": "21.3.0", + "version": "21.3.1", "description": "Add checksums to files in S3 which don't have one", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -45,12 +45,12 @@ "failFast": true }, "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0" }, "devDependencies": { - "@cumulus/schemas": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/schemas": "21.3.1", + "@cumulus/types": "21.3.1", "@types/aws-lambda": "^8.10.58" } } diff --git a/tasks/add-unique-granuleID/package.json b/tasks/add-unique-granuleID/package.json index f1f7ef39d640..2ec272d4c6c5 100644 --- a/tasks/add-unique-granuleID/package.json +++ b/tasks/add-unique-granuleID/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/add-unique-granule-id", - "version": "21.3.0", + "version": "21.3.1", "description": "A Cumulus task to add a unique granule ID to a granule", "main": "dist/index.js", "private": true, @@ -40,11 +40,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "21.3.0", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/types": "21.3.0" + "@cumulus/ingest": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/types": "21.3.1" } } diff --git a/tasks/change-granule-collection-pg/package.json b/tasks/change-granule-collection-pg/package.json index 480cbb435511..67d76c3503be 100644 --- a/tasks/change-granule-collection-pg/package.json +++ b/tasks/change-granule-collection-pg/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/change-granule-collection-pg", - "version": "21.3.0", + "version": "21.3.1", "description": "Move granule files and postgres records to a new collection", "private": true, "main": "dist/index.js", @@ -42,19 +42,19 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/types": "21.3.0" + "@cumulus/errors": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/types": "21.3.1" }, "devDependencies": { - "@cumulus/api": "21.3.0", - "@cumulus/db": "21.3.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/es-client": "21.3.0", + "@cumulus/api": "21.3.1", + "@cumulus/db": "21.3.1", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/es-client": "21.3.1", "lodash": "^4.17.21" } } diff --git a/tasks/change-granule-collection-s3/package.json b/tasks/change-granule-collection-s3/package.json index 0228f41f25dc..eadbae0ee496 100644 --- a/tasks/change-granule-collection-s3/package.json +++ b/tasks/change-granule-collection-s3/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/change-granule-collection-s3", - "version": "21.3.0", + "version": "21.3.1", "description": "Move granule files and postgres records to a new collection", "private": true, "main": "dist/index.js", @@ -42,20 +42,20 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/types": "21.3.1", "lodash": "^4.17.21" }, "devDependencies": { "@aws-sdk/client-sns": "^3.993.0", - "@cumulus/api": "21.3.0" + "@cumulus/api": "21.3.1" } } diff --git a/tasks/discover-granules/package.json b/tasks/discover-granules/package.json index bb088762b097..7a15de32da2b 100644 --- a/tasks/discover-granules/package.json +++ b/tasks/discover-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-granules", - "version": "21.3.0", + "version": "21.3.1", "description": "Discover Granules in FTP/HTTP/HTTPS/SFTP/S3 endpoints", "main": "index.js", "directories": { @@ -39,16 +39,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "21.3.0", + "@cumulus/api-client": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/ingest": "21.3.1", + "@cumulus/logger": "21.3.1", "got": "^11.8.5", "lodash": "^4.17.21", "p-map": "^4.0.0" }, "devDependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0" + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1" } } diff --git a/tasks/discover-pdrs/package.json b/tasks/discover-pdrs/package.json index 919333c03d2d..5f05971e7c17 100644 --- a/tasks/discover-pdrs/package.json +++ b/tasks/discover-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-pdrs", - "version": "21.3.0", + "version": "21.3.1", "description": "Discover PDRs in FTP and HTTP endpoints", "main": "index.js", "directories": { @@ -38,14 +38,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", + "@cumulus/ingest": "21.3.1", "lodash": "^4.17.21", "p-filter": "^2.1.0" }, "devDependencies": { - "@cumulus/common": "21.3.0", - "@cumulus/errors": "21.3.0" + "@cumulus/common": "21.3.1", + "@cumulus/errors": "21.3.1" } } diff --git a/tasks/files-to-granules/package.json b/tasks/files-to-granules/package.json index bc0c447ecb18..3f05ac40a037 100644 --- a/tasks/files-to-granules/package.json +++ b/tasks/files-to-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/files-to-granules", - "version": "21.3.0", + "version": "21.3.1", "description": "Converts array-of-files input into a granules object by extracting granuleId from filename", "main": "index.js", "directories": { @@ -36,13 +36,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0", + "@cumulus/errors": "21.3.1", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "21.3.0", - "@cumulus/schemas": "21.3.0" + "@cumulus/common": "21.3.1", + "@cumulus/schemas": "21.3.1" } } diff --git a/tasks/hello-world/package.json b/tasks/hello-world/package.json index 73e003398751..9e271376c331 100644 --- a/tasks/hello-world/package.json +++ b/tasks/hello-world/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hello-world", - "version": "21.3.0", + "version": "21.3.1", "description": "Example task", "main": "index.js", "directories": { @@ -36,8 +36,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0" } } diff --git a/tasks/hyrax-metadata-updates/package.json b/tasks/hyrax-metadata-updates/package.json index 40ffd839a04d..ec15ef1cf20c 100644 --- a/tasks/hyrax-metadata-updates/package.json +++ b/tasks/hyrax-metadata-updates/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hyrax-metadata-updates", - "version": "21.3.0", + "version": "21.3.1", "description": "Update granule metadata with hooks to OPeNDAP URL", "main": "index.js", "directories": { @@ -41,17 +41,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmr-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmr-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0", + "@cumulus/errors": "21.3.1", "lodash": "^4.17.21", "xml2js": "0.6.2" }, "devDependencies": { - "@cumulus/schemas": "21.3.0", + "@cumulus/schemas": "21.3.1", "jsonwebtoken": "^9.0.0", "nock": "^12.0.1", "rewire": "^6.0.0" diff --git a/tasks/lzards-backup/package.json b/tasks/lzards-backup/package.json index f71d8b7b3dc2..d3b77dc3384a 100644 --- a/tasks/lzards-backup/package.json +++ b/tasks/lzards-backup/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/lzards-backup", - "version": "21.3.0", + "version": "21.3.1", "description": "Run LZARDS backup", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -45,20 +45,20 @@ "failFast": true }, "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/db": "21.3.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/launchpad-auth": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/lzards-api-client": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/db": "21.3.1", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/launchpad-auth": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/lzards-api-client": "21.3.1", + "@cumulus/message": "21.3.1", "got": "^11.8.5" }, "devDependencies": { - "@cumulus/schemas": "21.3.0", - "@cumulus/types": "21.3.0" + "@cumulus/schemas": "21.3.1", + "@cumulus/types": "21.3.1" } } diff --git a/tasks/move-granules/package.json b/tasks/move-granules/package.json index 86f8546b52e9..bbac7c027ba0 100644 --- a/tasks/move-granules/package.json +++ b/tasks/move-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/move-granules", - "version": "21.3.0", + "version": "21.3.1", "description": "Move granule files from staging to final location", "main": "index.js", "directories": { @@ -41,18 +41,18 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/errors": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/errors": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/message": "21.3.1", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/schemas": "21.3.0" + "@cumulus/schemas": "21.3.1" } } diff --git a/tasks/orca-copy-to-archive-adapter/package.json b/tasks/orca-copy-to-archive-adapter/package.json index a144abfeb5c0..c83ff30b8054 100644 --- a/tasks/orca-copy-to-archive-adapter/package.json +++ b/tasks/orca-copy-to-archive-adapter/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/orca-copy-to-archive-adapter", - "version": "21.3.0", + "version": "21.3.1", "description": "Adapter to invoke orca copy-to-archive lambda", "main": "dist/index.js", "private": true, @@ -40,13 +40,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/logger": "21.3.0", + "@cumulus/logger": "21.3.1", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/schemas": "21.3.0" + "@cumulus/schemas": "21.3.1" } } diff --git a/tasks/orca-recovery-adapter/package.json b/tasks/orca-recovery-adapter/package.json index def847422609..6c0d189a678a 100644 --- a/tasks/orca-recovery-adapter/package.json +++ b/tasks/orca-recovery-adapter/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/orca-recovery-adapter", - "version": "21.3.0", + "version": "21.3.1", "description": "Adapter to invoke orca recovery workflow", "main": "dist/index.js", "private": true, @@ -40,14 +40,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/logger": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/logger": "21.3.1", + "@cumulus/message": "21.3.1", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/schemas": "21.3.0" + "@cumulus/schemas": "21.3.1" } } diff --git a/tasks/parse-pdr/package.json b/tasks/parse-pdr/package.json index 8a0a8208748f..ace75e5c00ea 100644 --- a/tasks/parse-pdr/package.json +++ b/tasks/parse-pdr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/parse-pdr", - "version": "21.3.0", + "version": "21.3.1", "description": "Download and Parse a given PDR", "license": "Apache-2.0", "main": "index.js", @@ -34,18 +34,18 @@ "failFast": true }, "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/collection-config-store": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/collection-config-store": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/logger": "21.3.0", - "@cumulus/pvl": "21.3.0", + "@cumulus/errors": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/logger": "21.3.1", + "@cumulus/pvl": "21.3.1", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/test-data": "21.3.0" + "@cumulus/test-data": "21.3.1" } } diff --git a/tasks/pdr-status-check/package.json b/tasks/pdr-status-check/package.json index 9584350d39e9..b0db2b3735bd 100644 --- a/tasks/pdr-status-check/package.json +++ b/tasks/pdr-status-check/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pdr-status-check", - "version": "21.3.0", + "version": "21.3.1", "description": "Checks execution status of granules in a PDR", "main": "index.js", "directories": { @@ -35,9 +35,9 @@ "failFast": true }, "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0" + "@cumulus/errors": "21.3.1" } } diff --git a/tasks/post-to-cmr/package.json b/tasks/post-to-cmr/package.json index a747c35347e6..433acaae4408 100644 --- a/tasks/post-to-cmr/package.json +++ b/tasks/post-to-cmr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/post-to-cmr", - "version": "21.3.0", + "version": "21.3.1", "description": "Post a given granule to CMR", "main": "index.js", "directories": { @@ -36,17 +36,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0", - "@cumulus/launchpad-auth": "21.3.0", + "@cumulus/errors": "21.3.1", + "@cumulus/launchpad-auth": "21.3.1", "lodash": "^4.17.21", "p-map": "^4.0.0" }, "devDependencies": { - "@cumulus/cmr-client": "21.3.0", - "@cumulus/schemas": "21.3.0" + "@cumulus/cmr-client": "21.3.1", + "@cumulus/schemas": "21.3.1" } } diff --git a/tasks/queue-granules/package.json b/tasks/queue-granules/package.json index c3ef882577a6..98fa35611823 100644 --- a/tasks/queue-granules/package.json +++ b/tasks/queue-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-granules", - "version": "21.3.0", + "version": "21.3.1", "description": "Add discovered granules to the queue", "main": "dist/src/index.js", "directories": { @@ -45,12 +45,12 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "21.3.0", - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api-client": "21.3.1", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/ingest": "21.3.1", + "@cumulus/message": "21.3.1", "lodash": "^4.17.21", "p-map": "^4.0.0" } diff --git a/tasks/queue-pdrs/package.json b/tasks/queue-pdrs/package.json index 02943a03ce1b..50dd327c2b08 100644 --- a/tasks/queue-pdrs/package.json +++ b/tasks/queue-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-pdrs", - "version": "21.3.0", + "version": "21.3.1", "description": "Add discovered PDRs to a queue", "main": "index.js", "directories": { @@ -35,11 +35,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/ingest": "21.3.1", + "@cumulus/message": "21.3.1", "lodash": "^4.17.21" } } diff --git a/tasks/queue-workflow/package.json b/tasks/queue-workflow/package.json index 64a1a0398308..f7326b6ebb5c 100644 --- a/tasks/queue-workflow/package.json +++ b/tasks/queue-workflow/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-workflow", - "version": "21.3.0", + "version": "21.3.1", "description": "Add workflow to the queue", "main": "index.js", "directories": { @@ -35,11 +35,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/message": "21.3.0", + "@cumulus/ingest": "21.3.1", + "@cumulus/message": "21.3.1", "lodash": "^4.17.21" } } diff --git a/tasks/send-pan/package.json b/tasks/send-pan/package.json index d4236d999e27..8a60b0e2efb9 100644 --- a/tasks/send-pan/package.json +++ b/tasks/send-pan/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/send-pan", - "version": "21.3.0", + "version": "21.3.1", "description": "Sends a PAN response after parsing a PDR.", "main": "dist/index.js", "private": true, @@ -40,15 +40,15 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/api": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/ingest": "21.3.1", + "@cumulus/logger": "21.3.1", "got": "^11.8.5" }, "devDependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "url-join": "^4.0.0" } } diff --git a/tasks/sf-sqs-report/package.json b/tasks/sf-sqs-report/package.json index 8e6a23d81478..89ad8c3e5520 100644 --- a/tasks/sf-sqs-report/package.json +++ b/tasks/sf-sqs-report/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sf-sqs-report", - "version": "21.3.0", + "version": "21.3.1", "description": "Sends an incoming Cumulus message to SQS", "main": "index.js", "directories": { @@ -36,11 +36,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "21.3.0" + "@cumulus/common": "21.3.1" } } diff --git a/tasks/sync-granule/package.json b/tasks/sync-granule/package.json index 285b7c583de9..359658738be3 100644 --- a/tasks/sync-granule/package.json +++ b/tasks/sync-granule/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sync-granule", - "version": "21.3.0", + "version": "21.3.1", "description": "Download a given granule", "main": "index.js", "directories": { @@ -40,21 +40,21 @@ "failFast": true }, "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/collection-config-store": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/collection-config-store": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/errors": "21.3.0", - "@cumulus/ingest": "21.3.0", - "@cumulus/message": "21.3.0", - "@cumulus/types": "21.3.0", + "@cumulus/errors": "21.3.1", + "@cumulus/ingest": "21.3.1", + "@cumulus/message": "21.3.1", + "@cumulus/types": "21.3.1", "lodash": "^4.17.21", "p-map": "^2.1.0", "uuid": "^3.4.0" }, "devDependencies": { - "@cumulus/schemas": "21.3.0", - "@cumulus/sftp-client": "21.3.0", - "@cumulus/test-data": "21.3.0" + "@cumulus/schemas": "21.3.1", + "@cumulus/sftp-client": "21.3.1", + "@cumulus/test-data": "21.3.1" } } diff --git a/tasks/test-processing/package.json b/tasks/test-processing/package.json index 9f003c50f32c..09568f895231 100644 --- a/tasks/test-processing/package.json +++ b/tasks/test-processing/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-processing", - "version": "21.3.0", + "version": "21.3.1", "description": "Fake processing task used for integration tests", "main": "index.js", "homepage": "https://github.com/nasa/cumulus/tree/master/tasks/test-processing", @@ -24,8 +24,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", + "@cumulus/aws-client": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/integration-tests": "21.3.0" + "@cumulus/integration-tests": "21.3.1" } } diff --git a/tasks/update-cmr-access-constraints/package.json b/tasks/update-cmr-access-constraints/package.json index ee0446ec098b..e8c6460d3972 100644 --- a/tasks/update-cmr-access-constraints/package.json +++ b/tasks/update-cmr-access-constraints/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/update-cmr-access-constraints", - "version": "21.3.0", + "version": "21.3.1", "description": "Updates CMR metadata to set access constraints", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -37,13 +37,13 @@ "failFast": true }, "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", "lodash": "^4.17.5" }, "devDependencies": { - "@cumulus/common": "21.3.0", - "@cumulus/schemas": "21.3.0" + "@cumulus/common": "21.3.1", + "@cumulus/schemas": "21.3.1" } } diff --git a/tasks/update-granules-cmr-metadata-file-links/package.json b/tasks/update-granules-cmr-metadata-file-links/package.json index c466e02870f3..b39ae52d1df2 100644 --- a/tasks/update-granules-cmr-metadata-file-links/package.json +++ b/tasks/update-granules-cmr-metadata-file-links/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/update-granules-cmr-metadata-file-links", - "version": "21.3.0", + "version": "21.3.1", "description": "Update CMR metadata files with correct online access urls and etags and transfer etag info to granules' CMR files", "main": "index.js", "directories": { @@ -40,15 +40,15 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0", - "@cumulus/cmrjs": "21.3.0", - "@cumulus/common": "21.3.0", + "@cumulus/aws-client": "21.3.1", + "@cumulus/cmrjs": "21.3.1", + "@cumulus/common": "21.3.1", "@cumulus/cumulus-message-adapter-js": "2.4.0", - "@cumulus/distribution-utils": "21.3.0", - "@cumulus/logger": "21.3.0", + "@cumulus/distribution-utils": "21.3.1", + "@cumulus/logger": "21.3.1", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/schemas": "21.3.0" + "@cumulus/schemas": "21.3.1" } } diff --git a/tf-modules/ingest/package.json b/tf-modules/ingest/package.json index 32affcc8cf28..80769122ca7d 100644 --- a/tf-modules/ingest/package.json +++ b/tf-modules/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest-module", - "version": "21.3.0", + "version": "21.3.1", "description": "Terraform module for data ingest related functionality", "engines": { "node": ">=22.21.1" diff --git a/tf-modules/internal/cumulus-test-cleanup/package.json b/tf-modules/internal/cumulus-test-cleanup/package.json index c005ede9165c..2201f34bba97 100644 --- a/tf-modules/internal/cumulus-test-cleanup/package.json +++ b/tf-modules/internal/cumulus-test-cleanup/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cumulus-test-cleanup", - "version": "21.3.0", + "version": "21.3.1", "description": "Nightly cron job for cleaning up integration test artifacts", "main": "index.js", "engines": { diff --git a/tf-modules/s3-replicator/package.json b/tf-modules/s3-replicator/package.json index 16a141380c7d..a21039c56b57 100644 --- a/tf-modules/s3-replicator/package.json +++ b/tf-modules/s3-replicator/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/s3-replicator", - "version": "21.3.0", + "version": "21.3.1", "description": "Replicate S3 Events to alternate bucket. Solves same-region replication.", "engines": { "node": ">=22.21.1" @@ -26,10 +26,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "21.3.0" + "@cumulus/aws-client": "21.3.1" }, "devDependencies": { - "@cumulus/common": "21.3.0" + "@cumulus/common": "21.3.1" }, "private": true } diff --git a/website/package.json b/website/package.json index 60a2ab8d438a..fbdd1293611d 100644 --- a/website/package.json +++ b/website/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/website", - "version": "21.3.0", + "version": "21.3.1", "description": "The Cumulus website", "license": "Apache-2.0", "scripts": { From e92496ef48b9ed51c008ab85461b3bb71b8303c7 Mon Sep 17 00:00:00 2001 From: Mike Dorfman <42116953+mikedorfman@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:19:44 -0500 Subject: [PATCH 14/26] mikedorfman/CUMULUS-4542 (#4258) * First pass at an aws-api-proxy task * Added a readme Updated schema generators * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated Changelog * Updated old granule-invalidator references * Added partial failure compatibility * Added package_uv to bin root from a different feature branch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updates based on PR feedback * Updated ajv from 6.12.3 to 8.18.0 to test dependency vulnerability fix * Updated readonly->readOnly in schemas after strict-mode check * Updates per PR feedback * Reverting config validation to use a static list of literals with a bit comment disclaimer * Fixed broken output validation * Reverted ajv version bump --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 + tasks/aws-api-proxy/README.md | 147 +++ tasks/aws-api-proxy/package.json | 31 + tasks/aws-api-proxy/pyproject.toml | 32 + .../src/aws_api_proxy/__init__.py | 14 + .../src/aws_api_proxy/aws_api_proxy.py | 89 ++ .../aws_api_proxy/schemas/config_schema.json | 109 ++ .../aws_api_proxy/schemas/config_schema.py | 82 ++ .../aws_api_proxy/schemas/output_schema.json | 18 + .../aws_api_proxy/schemas/output_schema.py | 40 + tasks/aws-api-proxy/src/main.py | 48 + tasks/aws-api-proxy/tests/conftest.py | 38 + .../aws_api_proxy/test_lambda_handler.py | 115 +++ .../aws_api_proxy/test_aws_api_proxy.py | 140 +++ tasks/aws-api-proxy/uv.lock | 958 ++++++++++++++++++ tasks/granule-invalidator/pyproject.toml | 3 + tasks/granule-invalidator/uv.lock | 250 +++++ 17 files changed, 2118 insertions(+) create mode 100644 tasks/aws-api-proxy/README.md create mode 100644 tasks/aws-api-proxy/package.json create mode 100644 tasks/aws-api-proxy/pyproject.toml create mode 100644 tasks/aws-api-proxy/src/aws_api_proxy/__init__.py create mode 100644 tasks/aws-api-proxy/src/aws_api_proxy/aws_api_proxy.py create mode 100644 tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.json create mode 100644 tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.py create mode 100644 tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.json create mode 100644 tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.py create mode 100644 tasks/aws-api-proxy/src/main.py create mode 100644 tasks/aws-api-proxy/tests/conftest.py create mode 100644 tasks/aws-api-proxy/tests/integration_tests/aws_api_proxy/test_lambda_handler.py create mode 100644 tasks/aws-api-proxy/tests/unit_tests/aws_api_proxy/test_aws_api_proxy.py create mode 100644 tasks/aws-api-proxy/uv.lock diff --git a/CHANGELOG.md b/CHANGELOG.md index b29c4449062f..96555f2e1c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added python code for CnmResponse task adapted from https://github.com/podaac/cumulus-cnm-response-task - **CUMULUS-4395** - Added supporting Terraform for the CnmResponse task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus. +- **CUMULUS-4498** + - Added `states:StartExecution` action to the `-steprole` IAM role. +- **CUMULUS-4542** + - Created the `aws-api-proxy` coreified task, which provides the functionality to post a list of CNM messages to a specified SNS topic. - **CUMULUS-4517** - Added the `@cumulus/db/s3search` module to enable Cumulus record search via S3-backed tables. The S3Search subclasses inherit from search/BaseSearch, allowing them to reuse existing query diff --git a/tasks/aws-api-proxy/README.md b/tasks/aws-api-proxy/README.md new file mode 100644 index 000000000000..3909f9b9030e --- /dev/null +++ b/tasks/aws-api-proxy/README.md @@ -0,0 +1,147 @@ + +# @cumulus/aws-api-proxy-task + +This task provides a Cumulus Message Adapter (CMA) wrapped proxy that invokes a +pre-approved AWS API action via boto3. It is intended for use within a Cumulus +workflow and enforces guardrails through schema validation and IAM permissions. +Currently, the schema allows the SNS `publish` action, with support for multiple +invocations in a single task execution via `iterate_by`. + +## Usage + +This lambda takes the following input and config objects, derived from workflow +configuration using the +[Cumulus Message Adapter](https://github.com/nasa/cumulus-message-adapter/blob/master/CONTRACT.md). +The output from the task follows the CMA contract and provides the information +detailed below. + +### Configuration + +| field name | type | default | required | values | description +| ---------- | ---- | ------- | -------- | ------ | ----------- +| service | string | N/A | yes | `sns` | AWS service to invoke +| action | string | N/A | yes | `publish` | Service action to invoke +| parameters | object | N/A | yes | N/A | Parameters passed to the AWS SDK action +| parameters.TopicArn | string | N/A | yes | N/A | SNS Topic ARN to publish to +| parameters.Message | string | N/A | yes | N/A | Message string to publish +| iterate_by | string | N/A | no | N/A | Name of a `parameters` field containing a list; each value produces a separate call +| parameter_filters | array | [] | no | N/A | List of transformations to apply to parameter fields before invoking the API +| parameter_filters[].name | string | N/A | yes | `json.dumps` | Filter to apply +| parameter_filters[].field | string | N/A | yes | N/A | Parameter field name to transform + +The following sample configuration publishes two messages to two topics by +iterating over a list of ARNs: + +```json +{ + "service": "sns", + "action": "publish", + "iterate_by": "TopicArn", + "parameters": { + "TopicArn": [ + "arn:aws:sns:us-east-1:123456789012:ExampleTopicA", + "arn:aws:sns:us-east-1:123456789012:ExampleTopicB" + ], + "Message": "Message for topic" + } +} +``` + +When `iterate_by` is set, the named field in `parameters` must be a list. The +task will create one API call per list entry by substituting each value into the +specified field. If `parameter_filters` are provided, each filter is applied to +the named field across all generated parameter sets. The only supported filter +is `json.dumps`, which serializes the field value as JSON. + +### Input + +The input payload for this task is expected to be empty. All invocation details +are passed via the task configuration. + +### Output + +The task returns a list of boto3 responses (one per entry generated by +`iterate_by`). +Each response includes the `MessageId` for the SNS publish request. Example: + +```json +{ + "result_list": [ + { + "MessageId": "11111111-1111-1111-1111-111111111111" + }, + { + "MessageId": "22222222-2222-2222-2222-222222222222" + } + ] +} +``` + +### Example workflow configuration + +This task can be used to send workflow notifications via SNS in a controlled +and audited way. A sample task state could look like: + +```json +{ + "Publish Workflow Notification": { + "Type": "Task", + "Resource": "${aws_api_proxy_arn}", + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "service": "sns", + "action": "publish", + "parameter_filters": [ + { + "name": "json.dumps", + "field": "Message" + } + ], + "parameters": { + "TopicArn": "${sns_publish_arn}", + "Message.$": "$.meta.cnm" + } + } + } + }, + "ResultPath": null, + "End": true + } +} +``` + +## Architecture + +```mermaid +graph TB + A["AWS Lambda Handler
lambda_handler"] -->|calls| B["Lambda Adapter
lambda_adapter"] + B -->|invokes boto3 client| C["AWS SDK (boto3)"] + C -->|executes action| D["AWS Service API
(e.g., SNS publish)"] + D -->|responses| E["CMA Output
result_list"] +``` + +### Internal Dependencies + +This task relies on the Cumulus Message Adapter and requires an IAM role that +permits the configured AWS service action (for example, `sns:Publish` to specific +topics). Guardrails are enforced by schema validation and IAM permissions. + +### External Dependencies + + - boto3 + - https://github.com/nasa/cumulus-message-adapter + - https://github.com/nasa/cumulus-message-adapter-python + +## Contributing + +To make a contribution, please [see our Cumulus contributing guidelines](https://github.com/nasa/cumulus/blob/master/CONTRIBUTING.md) +and our documentation on [adding a task](https://nasa.github.io/cumulus/docs/adding-a-task) + +## About Cumulus + +Cumulus is a cloud-based data ingest, archive, distribution and management +prototype for NASA's future Earth science data streams. + +[Cumulus Documentation](https://nasa.github.io/cumulus) diff --git a/tasks/aws-api-proxy/package.json b/tasks/aws-api-proxy/package.json new file mode 100644 index 000000000000..37bedf78a73d --- /dev/null +++ b/tasks/aws-api-proxy/package.json @@ -0,0 +1,31 @@ +{ + "name": "@cumulus/aws-api-proxy-task", + "private": true, + "version": "21.2.0", + "description": "AWS API proxy task", + "main": "index.js", + "homepage": "https://github.com/nasa/cumulus/tree/master/tasks/aws-api-proxy", + "repository": { + "type": "git", + "url": "https://github.com/nasa/cumulus" + }, + "scripts": { + "test": "uv run pytest -q", + "lint": "uv run ruff check . && uv run ruff format --diff . && uv run mypy src", + "clean": "rm -rf dist && rm -rf .venv && mkdir dist", + "build": "uv sync", + "prepare": "npm run build", + "package": "npm run clean && ../../bin/package_uv.sh 3.12 ${PWD}", + "install-python-deps": "npm run build" + }, + "publishConfig": { + "access": "restricted" + }, + "nyc": { + "exclude": [ + "tests" + ] + }, + "author": "Cumulus Authors", + "license": "Apache-2.0" +} diff --git a/tasks/aws-api-proxy/pyproject.toml b/tasks/aws-api-proxy/pyproject.toml new file mode 100644 index 000000000000..9ee8ee2c242b --- /dev/null +++ b/tasks/aws-api-proxy/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "aws-api-proxy" +version = "0.1.0" +description = "Makes specified AWS API calls wrapped the Cumulus Message Adapter" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "cumulus-message-adapter>=2.0.5", + "cumulus-message-adapter-python>=2.4.0", +] + +[project.urls] +homepage = "https://github.com/nasa/cumulus/blob/master/tasks/aws-api-proxy" +documentation = "https://github.com/nasa/cumulus/blob/master/tasks/aws-api-proxy/README.md" +repository = "https://github.com/nasa/cumulus.git" + +[dependency-groups] +dev = [ + "mypy>=1.0.0", + "ruff~=0.14.0", + "pytest>=9.0.2", + "pytest-cov>=7.0.0", + "moto~=5.1", + "pydantic>=2.10.4", +] + +[tool.pytest.ini_options] +pythonpath = ["src"] + +[project.scripts] +generate-config-schema = "aws_api_proxy.schemas.config_schema:main" +generate-output-schema = "aws_api_proxy.schemas.output_schema:main" diff --git a/tasks/aws-api-proxy/src/aws_api_proxy/__init__.py b/tasks/aws-api-proxy/src/aws_api_proxy/__init__.py new file mode 100644 index 000000000000..72b11baab0b4 --- /dev/null +++ b/tasks/aws-api-proxy/src/aws_api_proxy/__init__.py @@ -0,0 +1,14 @@ +"""AWS API proxy task for Cumulus. + +This module provides a CMA-wrapped way to call a specified boto3 client within AWS +lambda. This may be called once or multiple times against a provided list. Guardrails +are provided via configuration validation which specifies a predefined set of allowed +services and actions in addition to a dedicated IAM role for this lambda. +""" + +import logging +import os + +from cumulus_logger import CumulusLogger + +LOGGER = CumulusLogger(__name__, level=int(os.environ.get("LOGLEVEL", logging.DEBUG))) diff --git a/tasks/aws-api-proxy/src/aws_api_proxy/aws_api_proxy.py b/tasks/aws-api-proxy/src/aws_api_proxy/aws_api_proxy.py new file mode 100644 index 000000000000..bee709f0b71e --- /dev/null +++ b/tasks/aws-api-proxy/src/aws_api_proxy/aws_api_proxy.py @@ -0,0 +1,89 @@ +"""AWS API proxy task for Cumulus. + +This module provides a CMA-wrapped way to call a specified boto3 client within AWS +lambda. This may be called once or multiple times against a provided list. Guardrails +are provided via configuration validation which specifies a predefined set of allowed +services and actions in addition to a dedicated IAM role for this lambda. +""" + +import asyncio +import json +from typing import Any + +import boto3 + +from . import LOGGER + +PARAMETER_FILTERS = { + "json.dumps": json.dumps, +} + + +async def run_with_limit(method, parameters_list, max_concurrency=5): + """Run the given method with the provided parameters, limiting concurrency.""" + semaphore = asyncio.Semaphore(max_concurrency) + + async def worker(parameters): + async with semaphore: + # Since boto3 is not async, run it in a thread + # There is also aiobotocore which allows doing real async for AWS API calls. + # However, I'm avoiding additional third-party libraries that aren't + # officially supported/endorsed by AWS. If this were more complicated, + # moving to aibotocore might make sense, but for now it's fairly + # straight-forward to do it this way. + return await asyncio.to_thread(method, **parameters) + + results = await asyncio.gather( + *(worker(parameters) for parameters in parameters_list), return_exceptions=True + ) + return results + + +def lambda_adapter(event: dict, _: Any) -> dict[str, Any]: + """Handle AWS API Proxy requests.""" + config = event.get("config", {}) + service = config.get("service") + action = config.get("action") + parameters = config.get("parameters") + iterate_by = config.get("iterate_by") + parameter_filters = config.get("parameter_filters", []) + if iterate_by: + iterate_by_parameters = parameters.get(iterate_by) + if not isinstance(iterate_by_parameters, list): + raise ValueError( + f"iterate_by field '{iterate_by}' must be a list in parameters." + ) + parameters_list = [ + {**parameters, iterate_by: value} for value in parameters[iterate_by] + ] + else: + parameters_list = [parameters] + + for parameter_filter in parameter_filters: + parameter_filter_name = parameter_filter.get("name") + parameter_filter_field = parameter_filter.get("field") + parameter_filter_func = PARAMETER_FILTERS.get( + parameter_filter_name, lambda x: x + ) + parameters_list = [ + { + k: parameter_filter_func(v) if k == parameter_filter_field else v + for k, v in parameters.items() + } + for parameters in parameters_list + ] + + LOGGER.info( + f"Received request to call AWS service {service} " + f"with action {action} and parameters {parameters_list}" + ) + client = boto3.client(service) + method = getattr(client, action) + + responses = asyncio.run(run_with_limit(method, parameters_list)) + LOGGER.info( + f"Received response from AWS service {service} " + f"with action {action}: {responses}" + ) + + return {"result_list": responses} diff --git a/tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.json b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.json new file mode 100644 index 000000000000..3446dc91c47f --- /dev/null +++ b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.json @@ -0,0 +1,109 @@ +{ + "$defs": { + "ParameterFilter": { + "description": "Schema for parameter filters.", + "properties": { + "name": { + "const": "json.dumps", + "title": "Name", + "type": "string" + }, + "field": { + "title": "Field", + "type": "string" + } + }, + "required": [ + "name", + "field" + ], + "title": "ParameterFilter", + "type": "object" + }, + "SNSPublishParameters": { + "description": "Schema for SNS publish parameters.", + "properties": { + "TopicArn": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "title": "Topicarn" + }, + "Message": { + "anyOf": [ + {}, + { + "items": {}, + "type": "array" + } + ], + "title": "Message" + } + }, + "required": [ + "TopicArn", + "Message" + ], + "title": "SNSPublishParameters", + "type": "object" + } + }, + "description": "Configuration schema for AWS API proxy task.", + "properties": { + "service": { + "const": "sns", + "title": "Service", + "type": "string" + }, + "action": { + "const": "publish", + "title": "Action", + "type": "string" + }, + "parameters": { + "$ref": "#/$defs/SNSPublishParameters" + }, + "iterate_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Iterate By" + }, + "parameter_filters": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/ParameterFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parameter Filters" + } + }, + "required": [ + "service", + "action", + "parameters" + ], + "title": "Model", + "type": "object" +} diff --git a/tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.py b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.py new file mode 100644 index 000000000000..8008045b4d78 --- /dev/null +++ b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/config_schema.py @@ -0,0 +1,82 @@ +"""Pydantic schemas for task configuration.""" + +import argparse +import json +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, model_validator + + +class SNSPublishParameters(BaseModel): + """Schema for SNS publish parameters.""" + + TopicArn: str | list[str] + Message: Any | list[Any] + + +class ParameterFilter(BaseModel): + """Schema for parameter filters.""" + + # This list must always match the keys in PARAMETER_FILTERS in ..aws_api_proxy + # We cannot reference this list since Literals must be statically analyzable and + # PARAMETER_FILTERS is a runtime dict; https://typing.python.org/en/latest/spec/literal.html#illegal-parameters-for-literal-at-type-check-time + name: Literal[("json.dumps")] + field: str + + +class Model(BaseModel): + """Configuration schema for AWS API proxy task.""" + + service: Literal["sns"] + action: Literal["publish"] + parameters: SNSPublishParameters + iterate_by: str | None = None + parameter_filters: list[ParameterFilter] | None = None + + @model_validator(mode="after") + def _validate_iterate_by(self): + if self.iterate_by: + # Check if the field exists in parameters + if self.iterate_by not in self.parameters.model_fields: + raise TypeError( + f"iterate_by field '{self.iterate_by}' does not exist in " + f"parameters '{self.parameters.model_fields}'." + ) + + # Get the value + iterate_by_value = getattr(self.parameters, self.iterate_by) + + # Check if it's a list + if not isinstance(iterate_by_value, list): + raise TypeError( + f"iterate_by field '{self.iterate_by}' must reference a list field " + f"in parameters '{self.parameters.model_fields}'." + ) + return self + + +def save(output_dir: str) -> None: + """Print the JSON schema for the config model.""" + main_model_schema = Model.model_json_schema(mode="validation") + + output_path = Path(output_dir) / "config_schema.json" + output_path.write_text( + json.dumps(main_model_schema, indent=2) + "\n", encoding="utf-8" + ) + print(output_path) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate config schema JSON") + parser.add_argument( + "output_dir", + help="Directory to write config_schema.json", + ) + return parser.parse_args() + + +def main() -> None: + """Generate the config schema JSON file.""" + args = _parse_args() + save(args.output_dir) diff --git a/tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.json b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.json new file mode 100644 index 000000000000..8cf0b4859224 --- /dev/null +++ b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.json @@ -0,0 +1,18 @@ +{ + "description": "Output schema for the task.", + "properties": { + "result_list": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Result List", + "type": "array" + } + }, + "required": [ + "result_list" + ], + "title": "Model", + "type": "object" +} diff --git a/tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.py b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.py new file mode 100644 index 000000000000..0dea14b175e8 --- /dev/null +++ b/tasks/aws-api-proxy/src/aws_api_proxy/schemas/output_schema.py @@ -0,0 +1,40 @@ +"""Pydantic schemas for task output.""" + +import argparse +import json +from pathlib import Path +from typing import Any + +from pydantic import BaseModel + + +class Model(BaseModel): + """Output schema for the task.""" + + result_list: list[dict[str, Any]] + + +def save(output_dir: str) -> None: + """Print the JSON schema for the output model.""" + main_model_schema = Model.model_json_schema(mode="validation") + + output_path = Path(output_dir) / "output_schema.json" + output_path.write_text( + json.dumps(main_model_schema, indent=2) + "\n", encoding="utf-8" + ) + print(output_path) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate output schema JSON") + parser.add_argument( + "output_dir", + help="Directory to write output_schema.json", + ) + return parser.parse_args() + + +def main() -> None: + """Generate the output schema JSON file.""" + args = _parse_args() + save(args.output_dir) diff --git a/tasks/aws-api-proxy/src/main.py b/tasks/aws-api-proxy/src/main.py new file mode 100644 index 000000000000..239ea5c42b22 --- /dev/null +++ b/tasks/aws-api-proxy/src/main.py @@ -0,0 +1,48 @@ +"""aws-api-proxy Lambda function. + +This file provides the Lambda handler for the AWS API proxy task. +""" + +import logging +import os +from pathlib import Path +from typing import Any + +from aws_api_proxy.aws_api_proxy import lambda_adapter +from cumulus_logger import CumulusLogger +from run_cumulus_task import run_cumulus_task + +LOGGER = CumulusLogger(__name__, level=int(os.environ.get("LOGLEVEL", logging.DEBUG))) +SCHEMAS = { + "config": str( + Path(__file__).resolve().parent + / "aws_api_proxy" + / "schemas" + / "config_schema.json" + ), + "output": str( + Path(__file__).resolve().parent + / "aws_api_proxy" + / "schemas" + / "output_schema.json" + ), +} + + +def lambda_handler(event: dict, context: Any) -> Any: + """Lambda handler. + + AWS Lambda invokes this function when the Lambda is triggered. + Runs the task through the Cumulus Message Adapter (CMA). + + Args: + event: Lambda event input + context: Lambda context object + + Returns: + Output from the task wrapped by CMA + + """ + LOGGER.setMetadata(event, context) + cumulus_task_return = run_cumulus_task(lambda_adapter, event, context, SCHEMAS) + return cumulus_task_return diff --git a/tasks/aws-api-proxy/tests/conftest.py b/tasks/aws-api-proxy/tests/conftest.py new file mode 100644 index 000000000000..45e394a64321 --- /dev/null +++ b/tasks/aws-api-proxy/tests/conftest.py @@ -0,0 +1,38 @@ +import uuid + +import boto3 +import moto +import pytest + + +@pytest.fixture +def mock_sns(): + with moto.mock_aws(): + yield boto3.client("sns") + + +@pytest.fixture +def mock_sqs(): + with moto.mock_aws(): + yield boto3.client("sqs") + + +@pytest.fixture +def setup_sns_test(mock_sns, mock_sqs): + sns_topic = mock_sns.create_topic(Name=uuid.uuid4().hex) + topic_arn = sns_topic["TopicArn"] + + sqs_queue = mock_sqs.create_queue(QueueName=uuid.uuid4().hex) + queue_url = sqs_queue["QueueUrl"] + + attrs = mock_sqs.get_queue_attributes( + QueueUrl=queue_url, AttributeNames=["QueueArn"] + ) + queue_arn = attrs["Attributes"]["QueueArn"] + + mock_sns.subscribe( + TopicArn=topic_arn, + Protocol="sqs", + Endpoint=queue_arn, + ) + return sns_topic, sqs_queue diff --git a/tasks/aws-api-proxy/tests/integration_tests/aws_api_proxy/test_lambda_handler.py b/tasks/aws-api-proxy/tests/integration_tests/aws_api_proxy/test_lambda_handler.py new file mode 100644 index 000000000000..06c6b4786522 --- /dev/null +++ b/tasks/aws-api-proxy/tests/integration_tests/aws_api_proxy/test_lambda_handler.py @@ -0,0 +1,115 @@ +"""Tests for aws api proxy module lambda handler.""" + +import jsonschema +import pytest +from main import lambda_handler + + +def test_lambda_handler_prohibits_additional_parameters() -> None: + """Verify that lambda_adapter fails schema validation when unexpected parameters are + passed. This is explicitly tested to assure we're setting appropriate security + guardrails. + """ + event = { + "task_config": { + "service": "sns", + "action": "publish", + "parameters": { + "TopicArn": "abc", + "Message": "abc", + # Subject is not permitted in the schema + "Subject": "abc", + }, + } + } + with pytest.raises(jsonschema.exceptions.ValidationError): + lambda_handler(event, None) + + +def test_lambda_handler_prohibits_additional_actions() -> None: + """Verify that lambda_adapter fails schema validation when unexpected actions are + passed. This is explicitly tested to assure we're setting appropriate security + guardrails. + """ + event = { + "task_config": { + "service": "sns", + # list_topics is not a permitted action in the schema + "action": "list_topics", + "parameters": {}, + } + } + with pytest.raises(jsonschema.exceptions.ValidationError): + lambda_handler(event, None) + + +def test_lambda_handler_prohibits_additional_services() -> None: + """Verify that lambda_adapter fails schema validation when unexpected services are + passed. This is explicitly tested to assure we're setting appropriate security + guardrails. + """ + event = { + "task_config": { + # Secrets Manager is not a permitted service in the schema + "service": "secretsmanager", + "action": "get_secret_value", + "parameters": { + "SecretId": "abc", + "VersionId": "abc", + }, + } + } + with pytest.raises(jsonschema.exceptions.ValidationError): + lambda_handler(event, None) + + +def test_lambda_handler_no_filter_mapping() -> None: + """Verify an exception is raised when no filter match is found.""" + event = { + "task_config": { + "service": "sns", + "action": "publish", + "parameters": { + "TopicArn": "arn:aws:sns:us-east-1:123456789012:MyTopic", + "Message": [ + {"m": "first message"}, + {"m": "second message"}, + ], + }, + "iterate_by": "Message", + "parameter_filters": [ + { + "name": "nonexistent_filter", + "field": "Message", + } + ], + } + } + with pytest.raises(jsonschema.exceptions.ValidationError): + lambda_handler(event, None) + + +def test_lambda_handler_output_validation(mock_sns) -> None: + """Verify no exception is raised when validating nominal output.""" + topic = mock_sns.create_topic(Name="test-topic") + event = { + "task_config": { + "service": "sns", + "action": "publish", + "parameters": { + "TopicArn": topic["TopicArn"], + "Message": [ + {"m": "first message"}, + {"m": "second message"}, + ], + }, + "iterate_by": "Message", + "parameter_filters": [ + { + "name": "json.dumps", + "field": "Message", + } + ], + } + } + lambda_handler(event, None) diff --git a/tasks/aws-api-proxy/tests/unit_tests/aws_api_proxy/test_aws_api_proxy.py b/tasks/aws-api-proxy/tests/unit_tests/aws_api_proxy/test_aws_api_proxy.py new file mode 100644 index 000000000000..fcbf9edf72ed --- /dev/null +++ b/tasks/aws-api-proxy/tests/unit_tests/aws_api_proxy/test_aws_api_proxy.py @@ -0,0 +1,140 @@ +"""Tests for aws api proxy module.""" + +import asyncio +import json +import time + +import pytest +from aws_api_proxy.aws_api_proxy import lambda_adapter, run_with_limit + + +def test_lambda_adapter_list_publish(setup_sns_test, mock_sqs) -> None: + """Verify that lambda_adapter calls the AWS API with a list as expected.""" + sns_topic, sqs_queue = setup_sns_test + + messages = [ + {"m": "first message"}, + {"m": "second message"}, + ] + + event = { + "config": { + "service": "sns", + "action": "publish", + "parameters": { + "TopicArn": sns_topic["TopicArn"], + "Message": messages, + }, + "iterate_by": "Message", + "parameter_filters": [ + { + "name": "json.dumps", + "field": "Message", + } + ], + } + } + lambda_adapter_response = lambda_adapter(event, None) + responses = lambda_adapter_response.get("result_list") + assert len(responses) == len(messages) + assert all("MessageId" in response for response in responses) + + receive_message_response = mock_sqs.receive_message( + QueueUrl=sqs_queue["QueueUrl"], MaxNumberOfMessages=len(messages) + ) + + # The JSON we want is encoded within an encoded portion of the response + decoded_sqs_responses = [ + json.loads(m["Body"]) for m in receive_message_response.get("Messages") + ] + decoded_sqs_messages = [json.loads(m["Message"]) for m in decoded_sqs_responses] + + assert decoded_sqs_messages == messages + + +def test_lambda_adapter_iterate_by_nonexistent_field() -> None: + """Verify an exception is raised when the iterate_by field does not exist.""" + + event = { + "config": { + "service": "sns", + "action": "publish", + "parameters": { + "TopicArn": "arn:aws:sns:us-east-1:123456789012:MyTopic", + "Message": "abc", + }, + "iterate_by": "nonexistent_field", + } + } + with pytest.raises( + ValueError, + match="iterate_by field 'nonexistent_field' must be a list in parameters\\.", + ): + lambda_adapter(event, None) + + +def test_lambda_adapter_iterate_by_not_list() -> None: + """Verify an exception is raised when the iterate_by field is not a list.""" + + event = { + "config": { + "service": "sns", + "action": "publish", + "parameters": { + "TopicArn": "arn:aws:sns:us-east-1:123456789012:MyTopic", + "Message": "abc", + }, + "iterate_by": "Message", + } + } + with pytest.raises(ValueError): + lambda_adapter(event, None) + + +def test_run_with_limit_respects_max_concurrency() -> None: + """Verify run_with_limit caps concurrent calls and runs calls in parallel.""" + + concurrency = 2 + count = 10 + sleep_time = 0.5 + expected_total_time = (count / concurrency) * sleep_time + + # Add this buffer to the expected time to account for any overhead in scheduling + # tasks and running them in parallel. + extra_time = 0.1 + + def blocking_task(): + time.sleep(sleep_time) + + start_time = time.time() + results = asyncio.run( + run_with_limit(blocking_task, [{}] * count, max_concurrency=2) + ) + total_time = time.time() - start_time + + # We should get a result for each call + assert len(results) == count + + # If the total time is less, our semaphore/parallelism limit isn't working + assert total_time >= expected_total_time + + # If the total time is significantly more, we may not be running in parallel + assert total_time <= expected_total_time + extra_time + + +def test_run_with_limit_partial_failure() -> None: + """Verify run_with_limit caps concurrent calls and runs calls in parallel.""" + + count = 10 + cutoff = count / 2 + + def task(num): + if num >= cutoff: + raise ValueError("Simulated failure") + + results = asyncio.run( + run_with_limit(task, [{"num": i} for i in range(count)], max_concurrency=2) + ) + assert len(results) == count + assert len([r for r in results if isinstance(r, Exception)]) == count - cutoff + assert len([r for r in results if r is None]) == count - cutoff diff --git a/tasks/aws-api-proxy/uv.lock b/tasks/aws-api-proxy/uv.lock new file mode 100644 index 000000000000..34da00658505 --- /dev/null +++ b/tasks/aws-api-proxy/uv.lock @@ -0,0 +1,958 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "attrs" +version = "25.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, +] + +[[package]] +name = "aws-api-proxy" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "cumulus-message-adapter" }, + { name = "cumulus-message-adapter-python" }, +] + +[package.dev-dependencies] +dev = [ + { name = "moto" }, + { name = "mypy" }, + { name = "pydantic" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "cumulus-message-adapter", specifier = ">=2.0.5" }, + { name = "cumulus-message-adapter-python", specifier = ">=2.4.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "moto", specifier = "~=5.1" }, + { name = "mypy", specifier = ">=1.0.0" }, + { name = "pydantic", specifier = ">=2.10.4" }, + { name = "pytest", specifier = ">=9.0.2" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "ruff", specifier = "~=0.14.0" }, +] + +[[package]] +name = "boto3" +version = "1.40.76" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/04/8cf6cf7e6390c71b9c958f3bfedc45d1182b51a35f7789354bf7b2ff4e8c/boto3-1.40.76.tar.gz", hash = "sha256:16f4cf97f8dd8e0aae015f4dc66219bd7716a91a40d1e2daa0dafa241a4761c5", size = 111598, upload-time = "2025-11-18T20:23:10.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/8e/966263696eb441e8d1c4daa5fdfb3b4be10a96a23c418cc74c80b0b03d4e/boto3-1.40.76-py3-none-any.whl", hash = "sha256:8df6df755727be40ad9e309cfda07f9a12c147e17b639430c55d4e4feee8a167", size = 139359, upload-time = "2025-11-18T20:23:08.75Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.76" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/eb/50e2d280589a3c20c3b649bb66262d2b53a25c03262e4cc492048ac7540a/botocore-1.40.76.tar.gz", hash = "sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc", size = 14494001, upload-time = "2025-11-18T20:22:59.131Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/6c/522e05388aa6fc66cf8ea46c6b29809a1a6f527ea864998b01ffb368ca36/botocore-1.40.76-py3-none-any.whl", hash = "sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4", size = 14161738, upload-time = "2025-11-18T20:22:55.332Z" }, +] + +[[package]] +name = "certifi" +version = "2024.7.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/02/a95f2b11e207f68bc64d7aae9666fed2e2b3f307748d5123dffb72a1bbea/certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", size = 164065, upload-time = "2024-07-04T01:36:11.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/d5/c84e1a17bf61d4df64ca866a1c9a913874b4e9bdc131ec689a0ad013fb36/certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90", size = 162960, upload-time = "2024-07-04T01:36:09.038Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/f9/e92df5e07f3fc8d4c7f9a0f146ef75446bf870351cd37b788cf5897f8079/coverage-7.13.1.tar.gz", hash = "sha256:b7593fe7eb5feaa3fbb461ac79aac9f9fc0387a5ca8080b0c6fe2ca27b091afd", size = 825862, upload-time = "2025-12-28T15:42:56.969Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8a/87af46cccdfa78f53db747b09f5f9a21d5fc38d796834adac09b30a8ce74/coverage-7.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6f34591000f06e62085b1865c9bc5f7858df748834662a51edadfd2c3bfe0dd3", size = 218927, upload-time = "2025-12-28T15:40:52.814Z" }, + { url = "https://files.pythonhosted.org/packages/82/a8/6e22fdc67242a4a5a153f9438d05944553121c8f4ba70cb072af4c41362e/coverage-7.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b67e47c5595b9224599016e333f5ec25392597a89d5744658f837d204e16c63e", size = 219288, upload-time = "2025-12-28T15:40:54.262Z" }, + { url = "https://files.pythonhosted.org/packages/d0/0a/853a76e03b0f7c4375e2ca025df45c918beb367f3e20a0a8e91967f6e96c/coverage-7.13.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3e7b8bd70c48ffb28461ebe092c2345536fb18bbbf19d287c8913699735f505c", size = 250786, upload-time = "2025-12-28T15:40:56.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b4/694159c15c52b9f7ec7adf49d50e5f8ee71d3e9ef38adb4445d13dd56c20/coverage-7.13.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c223d078112e90dc0e5c4e35b98b9584164bea9fbbd221c0b21c5241f6d51b62", size = 253543, upload-time = "2025-12-28T15:40:57.585Z" }, + { url = "https://files.pythonhosted.org/packages/96/b2/7f1f0437a5c855f87e17cf5d0dc35920b6440ff2b58b1ba9788c059c26c8/coverage-7.13.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:794f7c05af0763b1bbd1b9e6eff0e52ad068be3b12cd96c87de037b01390c968", size = 254635, upload-time = "2025-12-28T15:40:59.443Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d1/73c3fdb8d7d3bddd9473c9c6a2e0682f09fc3dfbcb9c3f36412a7368bcab/coverage-7.13.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0642eae483cc8c2902e4af7298bf886d605e80f26382124cddc3967c2a3df09e", size = 251202, upload-time = "2025-12-28T15:41:01.328Z" }, + { url = "https://files.pythonhosted.org/packages/66/3c/f0edf75dcc152f145d5598329e864bbbe04ab78660fe3e8e395f9fff010f/coverage-7.13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f5e772ed5fef25b3de9f2008fe67b92d46831bd2bc5bdc5dd6bfd06b83b316f", size = 252566, upload-time = "2025-12-28T15:41:03.319Z" }, + { url = "https://files.pythonhosted.org/packages/17/b3/e64206d3c5f7dcbceafd14941345a754d3dbc78a823a6ed526e23b9cdaab/coverage-7.13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:45980ea19277dc0a579e432aef6a504fe098ef3a9032ead15e446eb0f1191aee", size = 250711, upload-time = "2025-12-28T15:41:06.411Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ad/28a3eb970a8ef5b479ee7f0c484a19c34e277479a5b70269dc652b730733/coverage-7.13.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f18eca6028ffa62adbd185a8f1e1dd242f2e68164dba5c2b74a5204850b4cf", size = 250278, upload-time = "2025-12-28T15:41:08.285Z" }, + { url = "https://files.pythonhosted.org/packages/54/e3/c8f0f1a93133e3e1291ca76cbb63565bd4b5c5df63b141f539d747fff348/coverage-7.13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8dca5590fec7a89ed6826fce625595279e586ead52e9e958d3237821fbc750c", size = 252154, upload-time = "2025-12-28T15:41:09.969Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bf/9939c5d6859c380e405b19e736321f1c7d402728792f4c752ad1adcce005/coverage-7.13.1-cp312-cp312-win32.whl", hash = "sha256:ff86d4e85188bba72cfb876df3e11fa243439882c55957184af44a35bd5880b7", size = 221487, upload-time = "2025-12-28T15:41:11.468Z" }, + { url = "https://files.pythonhosted.org/packages/fa/dc/7282856a407c621c2aad74021680a01b23010bb8ebf427cf5eacda2e876f/coverage-7.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:16cc1da46c04fb0fb128b4dc430b78fa2aba8a6c0c9f8eb391fd5103409a6ac6", size = 222299, upload-time = "2025-12-28T15:41:13.386Z" }, + { url = "https://files.pythonhosted.org/packages/10/79/176a11203412c350b3e9578620013af35bcdb79b651eb976f4a4b32044fa/coverage-7.13.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d9bc218650022a768f3775dd7fdac1886437325d8d295d923ebcfef4892ad5c", size = 220941, upload-time = "2025-12-28T15:41:14.975Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a4/e98e689347a1ff1a7f67932ab535cef82eb5e78f32a9e4132e114bbb3a0a/coverage-7.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cb237bfd0ef4d5eb6a19e29f9e528ac67ac3be932ea6b44fb6cc09b9f3ecff78", size = 218951, upload-time = "2025-12-28T15:41:16.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/7cbfe2bdc6e2f03d6b240d23dc45fdaf3fd270aaf2d640be77b7f16989ab/coverage-7.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1dcb645d7e34dcbcc96cd7c132b1fc55c39263ca62eb961c064eb3928997363b", size = 219325, upload-time = "2025-12-28T15:41:18.609Z" }, + { url = "https://files.pythonhosted.org/packages/59/f6/efdabdb4929487baeb7cb2a9f7dac457d9356f6ad1b255be283d58b16316/coverage-7.13.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3d42df8201e00384736f0df9be2ced39324c3907607d17d50d50116c989d84cd", size = 250309, upload-time = "2025-12-28T15:41:20.629Z" }, + { url = "https://files.pythonhosted.org/packages/12/da/91a52516e9d5aea87d32d1523f9cdcf7a35a3b298e6be05d6509ba3cfab2/coverage-7.13.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa3edde1aa8807de1d05934982416cb3ec46d1d4d91e280bcce7cca01c507992", size = 252907, upload-time = "2025-12-28T15:41:22.257Z" }, + { url = "https://files.pythonhosted.org/packages/75/38/f1ea837e3dc1231e086db1638947e00d264e7e8c41aa8ecacf6e1e0c05f4/coverage-7.13.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9edd0e01a343766add6817bc448408858ba6b489039eaaa2018474e4001651a4", size = 254148, upload-time = "2025-12-28T15:41:23.87Z" }, + { url = "https://files.pythonhosted.org/packages/7f/43/f4f16b881aaa34954ba446318dea6b9ed5405dd725dd8daac2358eda869a/coverage-7.13.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:985b7836931d033570b94c94713c6dba5f9d3ff26045f72c3e5dbc5fe3361e5a", size = 250515, upload-time = "2025-12-28T15:41:25.437Z" }, + { url = "https://files.pythonhosted.org/packages/84/34/8cba7f00078bd468ea914134e0144263194ce849ec3baad187ffb6203d1c/coverage-7.13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ffed1e4980889765c84a5d1a566159e363b71d6b6fbaf0bebc9d3c30bc016766", size = 252292, upload-time = "2025-12-28T15:41:28.459Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a4/cffac66c7652d84ee4ac52d3ccb94c015687d3b513f9db04bfcac2ac800d/coverage-7.13.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8842af7f175078456b8b17f1b73a0d16a65dcbdc653ecefeb00a56b3c8c298c4", size = 250242, upload-time = "2025-12-28T15:41:30.02Z" }, + { url = "https://files.pythonhosted.org/packages/f4/78/9a64d462263dde416f3c0067efade7b52b52796f489b1037a95b0dc389c9/coverage-7.13.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ccd7a6fca48ca9c131d9b0a2972a581e28b13416fc313fb98b6d24a03ce9a398", size = 250068, upload-time = "2025-12-28T15:41:32.007Z" }, + { url = "https://files.pythonhosted.org/packages/69/c8/a8994f5fece06db7c4a97c8fc1973684e178599b42e66280dded0524ef00/coverage-7.13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0403f647055de2609be776965108447deb8e384fe4a553c119e3ff6bfbab4784", size = 251846, upload-time = "2025-12-28T15:41:33.946Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f7/91fa73c4b80305c86598a2d4e54ba22df6bf7d0d97500944af7ef155d9f7/coverage-7.13.1-cp313-cp313-win32.whl", hash = "sha256:549d195116a1ba1e1ae2f5ca143f9777800f6636eab917d4f02b5310d6d73461", size = 221512, upload-time = "2025-12-28T15:41:35.519Z" }, + { url = "https://files.pythonhosted.org/packages/45/0b/0768b4231d5a044da8f75e097a8714ae1041246bb765d6b5563bab456735/coverage-7.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:5899d28b5276f536fcf840b18b61a9fce23cc3aec1d114c44c07fe94ebeaa500", size = 222321, upload-time = "2025-12-28T15:41:37.371Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b8/bdcb7253b7e85157282450262008f1366aa04663f3e3e4c30436f596c3e2/coverage-7.13.1-cp313-cp313-win_arm64.whl", hash = "sha256:868a2fae76dfb06e87291bcbd4dcbcc778a8500510b618d50496e520bd94d9b9", size = 220949, upload-time = "2025-12-28T15:41:39.553Z" }, + { url = "https://files.pythonhosted.org/packages/70/52/f2be52cc445ff75ea8397948c96c1b4ee14f7f9086ea62fc929c5ae7b717/coverage-7.13.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67170979de0dacac3f3097d02b0ad188d8edcea44ccc44aaa0550af49150c7dc", size = 219643, upload-time = "2025-12-28T15:41:41.567Z" }, + { url = "https://files.pythonhosted.org/packages/47/79/c85e378eaa239e2edec0c5523f71542c7793fe3340954eafb0bc3904d32d/coverage-7.13.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f80e2bb21bfab56ed7405c2d79d34b5dc0bc96c2c1d2a067b643a09fb756c43a", size = 219997, upload-time = "2025-12-28T15:41:43.418Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9b/b1ade8bfb653c0bbce2d6d6e90cc6c254cbb99b7248531cc76253cb4da6d/coverage-7.13.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f83351e0f7dcdb14d7326c3d8d8c4e915fa685cbfdc6281f9470d97a04e9dfe4", size = 261296, upload-time = "2025-12-28T15:41:45.207Z" }, + { url = "https://files.pythonhosted.org/packages/1f/af/ebf91e3e1a2473d523e87e87fd8581e0aa08741b96265730e2d79ce78d8d/coverage-7.13.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb3f6562e89bad0110afbe64e485aac2462efdce6232cdec7862a095dc3412f6", size = 263363, upload-time = "2025-12-28T15:41:47.163Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8b/fb2423526d446596624ac7fde12ea4262e66f86f5120114c3cfd0bb2befa/coverage-7.13.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77545b5dcda13b70f872c3b5974ac64c21d05e65b1590b441c8560115dc3a0d1", size = 265783, upload-time = "2025-12-28T15:41:49.03Z" }, + { url = "https://files.pythonhosted.org/packages/9b/26/ef2adb1e22674913b89f0fe7490ecadcef4a71fa96f5ced90c60ec358789/coverage-7.13.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4d240d260a1aed814790bbe1f10a5ff31ce6c21bc78f0da4a1e8268d6c80dbd", size = 260508, upload-time = "2025-12-28T15:41:51.035Z" }, + { url = "https://files.pythonhosted.org/packages/ce/7d/f0f59b3404caf662e7b5346247883887687c074ce67ba453ea08c612b1d5/coverage-7.13.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d2287ac9360dec3837bfdad969963a5d073a09a85d898bd86bea82aa8876ef3c", size = 263357, upload-time = "2025-12-28T15:41:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b1/29896492b0b1a047604d35d6fa804f12818fa30cdad660763a5f3159e158/coverage-7.13.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:0d2c11f3ea4db66b5cbded23b20185c35066892c67d80ec4be4bab257b9ad1e0", size = 260978, upload-time = "2025-12-28T15:41:54.589Z" }, + { url = "https://files.pythonhosted.org/packages/48/f2/971de1238a62e6f0a4128d37adadc8bb882ee96afbe03ff1570291754629/coverage-7.13.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:3fc6a169517ca0d7ca6846c3c5392ef2b9e38896f61d615cb75b9e7134d4ee1e", size = 259877, upload-time = "2025-12-28T15:41:56.263Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fc/0474efcbb590ff8628830e9aaec5f1831594874360e3251f1fdec31d07a3/coverage-7.13.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d10a2ed46386e850bb3de503a54f9fe8192e5917fcbb143bfef653a9355e9a53", size = 262069, upload-time = "2025-12-28T15:41:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/88/4f/3c159b7953db37a7b44c0eab8a95c37d1aa4257c47b4602c04022d5cb975/coverage-7.13.1-cp313-cp313t-win32.whl", hash = "sha256:75a6f4aa904301dab8022397a22c0039edc1f51e90b83dbd4464b8a38dc87842", size = 222184, upload-time = "2025-12-28T15:41:59.763Z" }, + { url = "https://files.pythonhosted.org/packages/58/a5/6b57d28f81417f9335774f20679d9d13b9a8fb90cd6160957aa3b54a2379/coverage-7.13.1-cp313-cp313t-win_amd64.whl", hash = "sha256:309ef5706e95e62578cda256b97f5e097916a2c26247c287bbe74794e7150df2", size = 223250, upload-time = "2025-12-28T15:42:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/81/7c/160796f3b035acfbb58be80e02e484548595aa67e16a6345e7910ace0a38/coverage-7.13.1-cp313-cp313t-win_arm64.whl", hash = "sha256:92f980729e79b5d16d221038dbf2e8f9a9136afa072f9d5d6ed4cb984b126a09", size = 221521, upload-time = "2025-12-28T15:42:03.275Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8e/ba0e597560c6563fc0adb902fda6526df5d4aa73bb10adf0574d03bd2206/coverage-7.13.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:97ab3647280d458a1f9adb85244e81587505a43c0c7cff851f5116cd2814b894", size = 218996, upload-time = "2025-12-28T15:42:04.978Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8e/764c6e116f4221dc7aa26c4061181ff92edb9c799adae6433d18eeba7a14/coverage-7.13.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8f572d989142e0908e6acf57ad1b9b86989ff057c006d13b76c146ec6a20216a", size = 219326, upload-time = "2025-12-28T15:42:06.691Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a6/6130dc6d8da28cdcbb0f2bf8865aeca9b157622f7c0031e48c6cf9a0e591/coverage-7.13.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d72140ccf8a147e94274024ff6fd8fb7811354cf7ef88b1f0a988ebaa5bc774f", size = 250374, upload-time = "2025-12-28T15:42:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/82/2b/783ded568f7cd6b677762f780ad338bf4b4750205860c17c25f7c708995e/coverage-7.13.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d3c9f051b028810f5a87c88e5d6e9af3c0ff32ef62763bf15d29f740453ca909", size = 252882, upload-time = "2025-12-28T15:42:10.515Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b2/9808766d082e6a4d59eb0cc881a57fc1600eb2c5882813eefff8254f71b5/coverage-7.13.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f398ba4df52d30b1763f62eed9de5620dcde96e6f491f4c62686736b155aa6e4", size = 254218, upload-time = "2025-12-28T15:42:12.208Z" }, + { url = "https://files.pythonhosted.org/packages/44/ea/52a985bb447c871cb4d2e376e401116520991b597c85afdde1ea9ef54f2c/coverage-7.13.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:132718176cc723026d201e347f800cd1a9e4b62ccd3f82476950834dad501c75", size = 250391, upload-time = "2025-12-28T15:42:14.21Z" }, + { url = "https://files.pythonhosted.org/packages/7f/1d/125b36cc12310718873cfc8209ecfbc1008f14f4f5fa0662aa608e579353/coverage-7.13.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e549d642426e3579b3f4b92d0431543b012dcb6e825c91619d4e93b7363c3f9", size = 252239, upload-time = "2025-12-28T15:42:16.292Z" }, + { url = "https://files.pythonhosted.org/packages/6a/16/10c1c164950cade470107f9f14bbac8485f8fb8515f515fca53d337e4a7f/coverage-7.13.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:90480b2134999301eea795b3a9dbf606c6fbab1b489150c501da84a959442465", size = 250196, upload-time = "2025-12-28T15:42:18.54Z" }, + { url = "https://files.pythonhosted.org/packages/2a/c6/cd860fac08780c6fd659732f6ced1b40b79c35977c1356344e44d72ba6c4/coverage-7.13.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e825dbb7f84dfa24663dd75835e7257f8882629fc11f03ecf77d84a75134b864", size = 250008, upload-time = "2025-12-28T15:42:20.365Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/a8c58d3d38f82a5711e1e0a67268362af48e1a03df27c03072ac30feefcf/coverage-7.13.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:623dcc6d7a7ba450bbdbeedbaa0c42b329bdae16491af2282f12a7e809be7eb9", size = 251671, upload-time = "2025-12-28T15:42:22.114Z" }, + { url = "https://files.pythonhosted.org/packages/f0/bc/fd4c1da651d037a1e3d53e8cb3f8182f4b53271ffa9a95a2e211bacc0349/coverage-7.13.1-cp314-cp314-win32.whl", hash = "sha256:6e73ebb44dca5f708dc871fe0b90cf4cff1a13f9956f747cc87b535a840386f5", size = 221777, upload-time = "2025-12-28T15:42:23.919Z" }, + { url = "https://files.pythonhosted.org/packages/4b/50/71acabdc8948464c17e90b5ffd92358579bd0910732c2a1c9537d7536aa6/coverage-7.13.1-cp314-cp314-win_amd64.whl", hash = "sha256:be753b225d159feb397bd0bf91ae86f689bad0da09d3b301478cd39b878ab31a", size = 222592, upload-time = "2025-12-28T15:42:25.619Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c8/a6fb943081bb0cc926499c7907731a6dc9efc2cbdc76d738c0ab752f1a32/coverage-7.13.1-cp314-cp314-win_arm64.whl", hash = "sha256:228b90f613b25ba0019361e4ab81520b343b622fc657daf7e501c4ed6a2366c0", size = 221169, upload-time = "2025-12-28T15:42:27.629Z" }, + { url = "https://files.pythonhosted.org/packages/16/61/d5b7a0a0e0e40d62e59bc8c7aa1afbd86280d82728ba97f0673b746b78e2/coverage-7.13.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:60cfb538fe9ef86e5b2ab0ca8fc8d62524777f6c611dcaf76dc16fbe9b8e698a", size = 219730, upload-time = "2025-12-28T15:42:29.306Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2c/8881326445fd071bb49514d1ce97d18a46a980712b51fee84f9ab42845b4/coverage-7.13.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:57dfc8048c72ba48a8c45e188d811e5efd7e49b387effc8fb17e97936dde5bf6", size = 220001, upload-time = "2025-12-28T15:42:31.319Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d7/50de63af51dfa3a7f91cc37ad8fcc1e244b734232fbc8b9ab0f3c834a5cd/coverage-7.13.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3f2f725aa3e909b3c5fdb8192490bdd8e1495e85906af74fe6e34a2a77ba0673", size = 261370, upload-time = "2025-12-28T15:42:32.992Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/d31722f0ec918fd7453b2758312729f645978d212b410cd0f7c2aed88a94/coverage-7.13.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ee68b21909686eeb21dfcba2c3b81fee70dcf38b140dcd5aa70680995fa3aa5", size = 263485, upload-time = "2025-12-28T15:42:34.759Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7a/2c114fa5c5fc08ba0777e4aec4c97e0b4a1afcb69c75f1f54cff78b073ab/coverage-7.13.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:724b1b270cb13ea2e6503476e34541a0b1f62280bc997eab443f87790202033d", size = 265890, upload-time = "2025-12-28T15:42:36.517Z" }, + { url = "https://files.pythonhosted.org/packages/65/d9/f0794aa1c74ceabc780fe17f6c338456bbc4e96bd950f2e969f48ac6fb20/coverage-7.13.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916abf1ac5cf7eb16bc540a5bf75c71c43a676f5c52fcb9fe75a2bd75fb944e8", size = 260445, upload-time = "2025-12-28T15:42:38.646Z" }, + { url = "https://files.pythonhosted.org/packages/49/23/184b22a00d9bb97488863ced9454068c79e413cb23f472da6cbddc6cfc52/coverage-7.13.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:776483fd35b58d8afe3acbd9988d5de592ab6da2d2a865edfdbc9fdb43e7c486", size = 263357, upload-time = "2025-12-28T15:42:40.788Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bd/58af54c0c9199ea4190284f389005779d7daf7bf3ce40dcd2d2b2f96da69/coverage-7.13.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b6f3b96617e9852703f5b633ea01315ca45c77e879584f283c44127f0f1ec564", size = 260959, upload-time = "2025-12-28T15:42:42.808Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2a/6839294e8f78a4891bf1df79d69c536880ba2f970d0ff09e7513d6e352e9/coverage-7.13.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd63e7b74661fed317212fab774e2a648bc4bb09b35f25474f8e3325d2945cd7", size = 259792, upload-time = "2025-12-28T15:42:44.818Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c3/528674d4623283310ad676c5af7414b9850ab6d55c2300e8aa4b945ec554/coverage-7.13.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:933082f161bbb3e9f90d00990dc956120f608cdbcaeea15c4d897f56ef4fe416", size = 262123, upload-time = "2025-12-28T15:42:47.108Z" }, + { url = "https://files.pythonhosted.org/packages/06/c5/8c0515692fb4c73ac379d8dc09b18eaf0214ecb76ea6e62467ba7a1556ff/coverage-7.13.1-cp314-cp314t-win32.whl", hash = "sha256:18be793c4c87de2965e1c0f060f03d9e5aff66cfeae8e1dbe6e5b88056ec153f", size = 222562, upload-time = "2025-12-28T15:42:49.144Z" }, + { url = "https://files.pythonhosted.org/packages/05/0e/c0a0c4678cb30dac735811db529b321d7e1c9120b79bd728d4f4d6b010e9/coverage-7.13.1-cp314-cp314t-win_amd64.whl", hash = "sha256:0e42e0ec0cd3e0d851cb3c91f770c9301f48647cb2877cb78f74bdaa07639a79", size = 223670, upload-time = "2025-12-28T15:42:51.218Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5f/b177aa0011f354abf03a8f30a85032686d290fdeed4222b27d36b4372a50/coverage-7.13.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eaecf47ef10c72ece9a2a92118257da87e460e113b83cc0d2905cbbe931792b4", size = 221707, upload-time = "2025-12-28T15:42:53.034Z" }, + { url = "https://files.pythonhosted.org/packages/cc/48/d9f421cb8da5afaa1a64570d9989e00fb7955e6acddc5a12979f7666ef60/coverage-7.13.1-py3-none-any.whl", hash = "sha256:2016745cb3ba554469d02819d78958b571792bb68e31302610e898f80dd3a573", size = 210722, upload-time = "2025-12-28T15:42:54.901Z" }, +] + +[[package]] +name = "cryptography" +version = "43.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/ba/0664727028b37e249e73879348cc46d45c5c1a2a2e81e8166462953c5755/cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d", size = 686927, upload-time = "2024-09-03T20:04:20.788Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/28/b92c98a04ba762f8cdeb54eba5c4c84e63cac037a7c5e70117d337b15ad6/cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d", size = 6223222, upload-time = "2024-09-03T20:04:14.466Z" }, + { url = "https://files.pythonhosted.org/packages/33/13/1193774705783ba364121aa2a60132fa31a668b8ababd5edfa1662354ccd/cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062", size = 3794751, upload-time = "2024-09-03T20:04:16.725Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4b/39bb3c4c8cfb3e94e736b8d8859ce5c81536e91a1033b1d26770c4249000/cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962", size = 3981827, upload-time = "2024-09-03T20:03:55.035Z" }, + { url = "https://files.pythonhosted.org/packages/ce/dc/1471d4d56608e1013237af334b8a4c35d53895694fbb73882d1c4fd3f55e/cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277", size = 3780034, upload-time = "2024-09-03T20:03:58.972Z" }, + { url = "https://files.pythonhosted.org/packages/ad/43/7a9920135b0d5437cc2f8f529fa757431eb6a7736ddfadfdee1cc5890800/cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a", size = 3993407, upload-time = "2024-09-03T20:03:36.682Z" }, + { url = "https://files.pythonhosted.org/packages/cc/42/9ab8467af6c0b76f3d9b8f01d1cf25b9c9f3f2151f4acfab888d21c55a72/cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042", size = 3886457, upload-time = "2024-09-03T20:03:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/a4/65/430509e31700286ec02868a2457d2111d03ccefc20349d24e58d171ae0a7/cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494", size = 4081499, upload-time = "2024-09-03T20:03:32.522Z" }, + { url = "https://files.pythonhosted.org/packages/bb/18/a04b6467e6e09df8c73b91dcee8878f4a438a43a3603dc3cd6f8003b92d8/cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2", size = 2616504, upload-time = "2024-09-03T20:04:09.459Z" }, + { url = "https://files.pythonhosted.org/packages/cc/73/0eacbdc437202edcbdc07f3576ed8fb8b0ab79d27bf2c5d822d758a72faa/cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d", size = 3067456, upload-time = "2024-09-03T20:03:40.775Z" }, + { url = "https://files.pythonhosted.org/packages/8a/b6/bc54b371f02cffd35ff8dc6baba88304d7cf8e83632566b4b42e00383e03/cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d", size = 6225263, upload-time = "2024-09-03T20:03:43.181Z" }, + { url = "https://files.pythonhosted.org/packages/00/0e/8217e348a1fa417ec4c78cd3cdf24154f5e76fd7597343a35bd403650dfd/cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806", size = 3794368, upload-time = "2024-09-03T20:03:18.051Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ed/38b6be7254d8f7251fde8054af597ee8afa14f911da67a9410a45f602fc3/cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85", size = 3981750, upload-time = "2024-09-03T20:04:18.775Z" }, + { url = "https://files.pythonhosted.org/packages/64/f3/b7946c3887cf7436f002f4cbb1e6aec77b8d299b86be48eeadfefb937c4b/cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c", size = 3778925, upload-time = "2024-09-03T20:03:45.022Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7e/ebda4dd4ae098a0990753efbb4b50954f1d03003846b943ea85070782da7/cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1", size = 3993152, upload-time = "2024-09-03T20:03:30.108Z" }, + { url = "https://files.pythonhosted.org/packages/43/f6/feebbd78a3e341e3913846a3bb2c29d0b09b1b3af1573c6baabc2533e147/cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa", size = 3886392, upload-time = "2024-09-03T20:03:34.543Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4c/ab0b9407d5247576290b4fd8abd06b7f51bd414f04eef0f2800675512d61/cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4", size = 4082606, upload-time = "2024-09-03T20:03:27.836Z" }, + { url = "https://files.pythonhosted.org/packages/05/36/e532a671998d6fcfdb9122da16434347a58a6bae9465e527e450e0bc60a5/cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47", size = 2617948, upload-time = "2024-09-03T20:03:25.446Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c6/c09cee6968add5ff868525c3815e5dccc0e3c6e89eec58dc9135d3c40e88/cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb", size = 3070445, upload-time = "2024-09-03T20:03:21.179Z" }, +] + +[[package]] +name = "cumulus-message-adapter" +version = "2.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "jsonpath-ng" }, + { name = "jsonschema" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/85/425c372e138d850247511446e0f59ca5230b2535e1ed4a38767c9de5b12c/cumulus_message_adapter-2.0.5.tar.gz", hash = "sha256:815b957fde9a1cd35c6b0a8f1c71ed7181d73d608b7222c462e3d6f2bbdb0232", size = 19380, upload-time = "2025-09-15T15:45:34.599Z" } + +[[package]] +name = "cumulus-message-adapter-python" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cumulus-message-adapter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/92/8e97cc640181f72eb7b2fd6c0524b9c28720c5c06da264ed8e8fa0704ecf/cumulus_message_adapter_python-2.4.0.tar.gz", hash = "sha256:d53f8d5d01582ef5960a3c24846de5ea354abf1273f01002825c99f6b050e707", size = 22755, upload-time = "2025-09-15T18:53:51.024Z" } + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + +[[package]] +name = "jsonpath-ng" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ply" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/86/08646239a313f895186ff0a4573452038eed8c86f54380b3ebac34d32fb2/jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c", size = 37838, upload-time = "2024-10-11T15:41:42.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/5a/73ecb3d82f8615f32ccdadeb9356726d6cae3a4bbc840b437ceb95708063/jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6", size = 30105, upload-time = "2024-11-20T17:58:30.418Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.17.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pyrsistent" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/3d/ca032d5ac064dff543aa13c984737795ac81abc9fb130cd2fcff17cfabc7/jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d", size = 297785, upload-time = "2022-11-29T20:37:47.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/97/c698bd9350f307daad79dd740806e1a59becd693bd11443a0f531e3229b3/jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6", size = 90379, upload-time = "2022-11-29T20:37:45.842Z" }, +] + +[[package]] +name = "librt" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/3f/4ca7dd7819bf8ff303aca39c3c60e5320e46e766ab7f7dd627d3b9c11bdf/librt-0.8.0.tar.gz", hash = "sha256:cb74cdcbc0103fc988e04e5c58b0b31e8e5dd2babb9182b6f9490488eb36324b", size = 177306, upload-time = "2026-02-12T14:53:54.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/53/f3bc0c4921adb0d4a5afa0656f2c0fbe20e18e3e0295e12985b9a5dc3f55/librt-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:17269dd2745dbe8e42475acb28e419ad92dfa38214224b1b01020b8cac70b645", size = 66511, upload-time = "2026-02-12T14:52:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/89/4b/4c96357432007c25a1b5e363045373a6c39481e49f6ba05234bb59a839c1/librt-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f4617cef654fca552f00ce5ffdf4f4b68770f18950e4246ce94629b789b92467", size = 68628, upload-time = "2026-02-12T14:52:31.491Z" }, + { url = "https://files.pythonhosted.org/packages/47/16/52d75374d1012e8fc709216b5eaa25f471370e2a2331b8be00f18670a6c7/librt-0.8.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5cb11061a736a9db45e3c1293cfcb1e3caf205912dfa085734ba750f2197ff9a", size = 198941, upload-time = "2026-02-12T14:52:32.489Z" }, + { url = "https://files.pythonhosted.org/packages/fc/11/d5dd89e5a2228567b1228d8602d896736247424484db086eea6b8010bcba/librt-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4bb00bd71b448f16749909b08a0ff16f58b079e2261c2e1000f2bbb2a4f0a45", size = 210009, upload-time = "2026-02-12T14:52:33.634Z" }, + { url = "https://files.pythonhosted.org/packages/49/d8/fc1a92a77c3020ee08ce2dc48aed4b42ab7c30fb43ce488d388673b0f164/librt-0.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95a719a049f0eefaf1952673223cf00d442952273cbd20cf2ed7ec423a0ef58d", size = 224461, upload-time = "2026-02-12T14:52:34.868Z" }, + { url = "https://files.pythonhosted.org/packages/7f/98/eb923e8b028cece924c246104aa800cf72e02d023a8ad4ca87135b05a2fe/librt-0.8.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bd32add59b58fba3439d48d6f36ac695830388e3da3e92e4fc26d2d02670d19c", size = 217538, upload-time = "2026-02-12T14:52:36.078Z" }, + { url = "https://files.pythonhosted.org/packages/fd/67/24e80ab170674a1d8ee9f9a83081dca4635519dbd0473b8321deecddb5be/librt-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4f764b2424cb04524ff7a486b9c391e93f93dc1bd8305b2136d25e582e99aa2f", size = 225110, upload-time = "2026-02-12T14:52:37.301Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c7/6fbdcbd1a6e5243c7989c21d68ab967c153b391351174b4729e359d9977f/librt-0.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f04ca50e847abc486fa8f4107250566441e693779a5374ba211e96e238f298b9", size = 217758, upload-time = "2026-02-12T14:52:38.89Z" }, + { url = "https://files.pythonhosted.org/packages/4b/bd/4d6b36669db086e3d747434430073e14def032dd58ad97959bf7e2d06c67/librt-0.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9ab3a3475a55b89b87ffd7e6665838e8458e0b596c22e0177e0f961434ec474a", size = 218384, upload-time = "2026-02-12T14:52:40.637Z" }, + { url = "https://files.pythonhosted.org/packages/50/2d/afe966beb0a8f179b132f3e95c8dd90738a23e9ebdba10f89a3f192f9366/librt-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3e36a8da17134ffc29373775d88c04832f9ecfab1880470661813e6c7991ef79", size = 241187, upload-time = "2026-02-12T14:52:43.55Z" }, + { url = "https://files.pythonhosted.org/packages/02/d0/6172ea4af2b538462785ab1a68e52d5c99cfb9866a7caf00fdf388299734/librt-0.8.0-cp312-cp312-win32.whl", hash = "sha256:4eb5e06ebcc668677ed6389164f52f13f71737fc8be471101fa8b4ce77baeb0c", size = 54914, upload-time = "2026-02-12T14:52:44.676Z" }, + { url = "https://files.pythonhosted.org/packages/d4/cb/ceb6ed6175612a4337ad49fb01ef594712b934b4bc88ce8a63554832eb44/librt-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:0a33335eb59921e77c9acc05d0e654e4e32e45b014a4d61517897c11591094f8", size = 62020, upload-time = "2026-02-12T14:52:45.676Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7e/61701acbc67da74ce06ddc7ba9483e81c70f44236b2d00f6a4bfee1aacbf/librt-0.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:24a01c13a2a9bdad20997a4443ebe6e329df063d1978bbe2ebbf637878a46d1e", size = 52443, upload-time = "2026-02-12T14:52:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/6d/32/3edb0bcb4113a9c8bdcd1750663a54565d255027657a5df9d90f13ee07fa/librt-0.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7f820210e21e3a8bf8fde2ae3c3d10106d4de9ead28cbfdf6d0f0f41f5b12fa1", size = 66522, upload-time = "2026-02-12T14:52:48.219Z" }, + { url = "https://files.pythonhosted.org/packages/30/ab/e8c3d05e281f5d405ebdcc5bc8ab36df23e1a4b40ac9da8c3eb9928b72b9/librt-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4831c44b8919e75ca0dfb52052897c1ef59fdae19d3589893fbd068f1e41afbf", size = 68658, upload-time = "2026-02-12T14:52:50.351Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d3/74a206c47b7748bbc8c43942de3ed67de4c231156e148b4f9250869593df/librt-0.8.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:88c6e75540f1f10f5e0fc5e87b4b6c290f0e90d1db8c6734f670840494764af8", size = 199287, upload-time = "2026-02-12T14:52:51.938Z" }, + { url = "https://files.pythonhosted.org/packages/fa/29/ef98a9131cf12cb95771d24e4c411fda96c89dc78b09c2de4704877ebee4/librt-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9646178cd794704d722306c2c920c221abbf080fede3ba539d5afdec16c46dad", size = 210293, upload-time = "2026-02-12T14:52:53.128Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3e/89b4968cb08c53d4c2d8b02517081dfe4b9e07a959ec143d333d76899f6c/librt-0.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e1af31a710e17891d9adf0dbd9a5fcd94901a3922a96499abdbf7ce658f4e01", size = 224801, upload-time = "2026-02-12T14:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/6d/28/f38526d501f9513f8b48d78e6be4a241e15dd4b000056dc8b3f06ee9ce5d/librt-0.8.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:507e94f4bec00b2f590fbe55f48cd518a208e2474a3b90a60aa8f29136ddbada", size = 218090, upload-time = "2026-02-12T14:52:55.758Z" }, + { url = "https://files.pythonhosted.org/packages/02/ec/64e29887c5009c24dc9c397116c680caffc50286f62bd99c39e3875a2854/librt-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f1178e0de0c271231a660fbef9be6acdfa1d596803464706862bef6644cc1cae", size = 225483, upload-time = "2026-02-12T14:52:57.375Z" }, + { url = "https://files.pythonhosted.org/packages/ee/16/7850bdbc9f1a32d3feff2708d90c56fc0490b13f1012e438532781aa598c/librt-0.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:71fc517efc14f75c2f74b1f0a5d5eb4a8e06aa135c34d18eaf3522f4a53cd62d", size = 218226, upload-time = "2026-02-12T14:52:58.534Z" }, + { url = "https://files.pythonhosted.org/packages/1c/4a/166bffc992d65ddefa7c47052010a87c059b44a458ebaf8f5eba384b0533/librt-0.8.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0583aef7e9a720dd40f26a2ad5a1bf2ccbb90059dac2b32ac516df232c701db3", size = 218755, upload-time = "2026-02-12T14:52:59.701Z" }, + { url = "https://files.pythonhosted.org/packages/da/5d/9aeee038bcc72a9cfaaee934463fe9280a73c5440d36bd3175069d2cb97b/librt-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5d0f76fc73480d42285c609c0ea74d79856c160fa828ff9aceab574ea4ecfd7b", size = 241617, upload-time = "2026-02-12T14:53:00.966Z" }, + { url = "https://files.pythonhosted.org/packages/64/ff/2bec6b0296b9d0402aa6ec8540aa19ebcb875d669c37800cb43d10d9c3a3/librt-0.8.0-cp313-cp313-win32.whl", hash = "sha256:e79dbc8f57de360f0ed987dc7de7be814b4803ef0e8fc6d3ff86e16798c99935", size = 54966, upload-time = "2026-02-12T14:53:02.042Z" }, + { url = "https://files.pythonhosted.org/packages/08/8d/bf44633b0182996b2c7ea69a03a5c529683fa1f6b8e45c03fe874ff40d56/librt-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:25b3e667cbfc9000c4740b282df599ebd91dbdcc1aa6785050e4c1d6be5329ab", size = 62000, upload-time = "2026-02-12T14:53:03.822Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fd/c6472b8e0eac0925001f75e366cf5500bcb975357a65ef1f6b5749389d3a/librt-0.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:e9a3a38eb4134ad33122a6d575e6324831f930a771d951a15ce232e0237412c2", size = 52496, upload-time = "2026-02-12T14:53:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/13/79ebfe30cd273d7c0ce37a5f14dc489c5fb8b722a008983db2cfd57270bb/librt-0.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:421765e8c6b18e64d21c8ead315708a56fc24f44075059702e421d164575fdda", size = 66078, upload-time = "2026-02-12T14:53:06.085Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8f/d11eca40b62a8d5e759239a80636386ef88adecb10d1a050b38cc0da9f9e/librt-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:48f84830a8f8ad7918afd743fd7c4eb558728bceab7b0e38fd5a5cf78206a556", size = 68309, upload-time = "2026-02-12T14:53:07.121Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b4/f12ee70a3596db40ff3c88ec9eaa4e323f3b92f77505b4d900746706ec6a/librt-0.8.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9f09d4884f882baa39a7e36bbf3eae124c4ca2a223efb91e567381d1c55c6b06", size = 196804, upload-time = "2026-02-12T14:53:08.164Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7e/70dbbdc0271fd626abe1671ad117bcd61a9a88cdc6a10ccfbfc703db1873/librt-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:693697133c3b32aa9b27f040e3691be210e9ac4d905061859a9ed519b1d5a376", size = 206915, upload-time = "2026-02-12T14:53:09.333Z" }, + { url = "https://files.pythonhosted.org/packages/79/13/6b9e05a635d4327608d06b3c1702166e3b3e78315846373446cf90d7b0bf/librt-0.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5512aae4648152abaf4d48b59890503fcbe86e85abc12fb9b096fe948bdd816", size = 221200, upload-time = "2026-02-12T14:53:10.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/6c/e19a3ac53e9414de43a73d7507d2d766cd22d8ca763d29a4e072d628db42/librt-0.8.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:995d24caa6bbb34bcdd4a41df98ac6d1af637cfa8975cb0790e47d6623e70e3e", size = 214640, upload-time = "2026-02-12T14:53:12.342Z" }, + { url = "https://files.pythonhosted.org/packages/30/f0/23a78464788619e8c70f090cfd099cce4973eed142c4dccb99fc322283fd/librt-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b9aef96d7593584e31ef6ac1eb9775355b0099fee7651fae3a15bc8657b67b52", size = 221980, upload-time = "2026-02-12T14:53:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/03/32/38e21420c5d7aa8a8bd2c7a7d5252ab174a5a8aaec8b5551968979b747bf/librt-0.8.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4f6e975377fbc4c9567cb33ea9ab826031b6c7ec0515bfae66a4fb110d40d6da", size = 215146, upload-time = "2026-02-12T14:53:14.8Z" }, + { url = "https://files.pythonhosted.org/packages/bb/00/bd9ecf38b1824c25240b3ad982fb62c80f0a969e6679091ba2b3afb2b510/librt-0.8.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:daae5e955764be8fd70a93e9e5133c75297f8bce1e802e1d3683b98f77e1c5ab", size = 215203, upload-time = "2026-02-12T14:53:16.087Z" }, + { url = "https://files.pythonhosted.org/packages/b9/60/7559bcc5279d37810b98d4a52616febd7b8eef04391714fd6bdf629598b1/librt-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7bd68cebf3131bb920d5984f75fe302d758db33264e44b45ad139385662d7bc3", size = 237937, upload-time = "2026-02-12T14:53:17.236Z" }, + { url = "https://files.pythonhosted.org/packages/41/cc/be3e7da88f1abbe2642672af1dc00a0bccece11ca60241b1883f3018d8d5/librt-0.8.0-cp314-cp314-win32.whl", hash = "sha256:1e6811cac1dcb27ca4c74e0ca4a5917a8e06db0d8408d30daee3a41724bfde7a", size = 50685, upload-time = "2026-02-12T14:53:18.888Z" }, + { url = "https://files.pythonhosted.org/packages/38/27/e381d0df182a8f61ef1f6025d8b138b3318cc9d18ad4d5f47c3bf7492523/librt-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:178707cda89d910c3b28bf5aa5f69d3d4734e0f6ae102f753ad79edef83a83c7", size = 57872, upload-time = "2026-02-12T14:53:19.942Z" }, + { url = "https://files.pythonhosted.org/packages/c5/0c/ca9dfdf00554a44dea7d555001248269a4bab569e1590a91391feb863fa4/librt-0.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3e8b77b5f54d0937b26512774916041756c9eb3e66f1031971e626eea49d0bf4", size = 48056, upload-time = "2026-02-12T14:53:21.473Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ed/6cc9c4ad24f90c8e782193c7b4a857408fd49540800613d1356c63567d7b/librt-0.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:789911e8fa40a2e82f41120c936b1965f3213c67f5a483fc5a41f5839a05dcbb", size = 68307, upload-time = "2026-02-12T14:53:22.498Z" }, + { url = "https://files.pythonhosted.org/packages/84/d8/0e94292c6b3e00b6eeea39dd44d5703d1ec29b6dafce7eea19dc8f1aedbd/librt-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2b37437e7e4ef5e15a297b36ba9e577f73e29564131d86dd75875705e97402b5", size = 70999, upload-time = "2026-02-12T14:53:23.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f4/6be1afcbdeedbdbbf54a7c9d73ad43e1bf36897cebf3978308cd64922e02/librt-0.8.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:671a6152edf3b924d98a5ed5e6982ec9cb30894085482acadce0975f031d4c5c", size = 220782, upload-time = "2026-02-12T14:53:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/f0/8d/f306e8caa93cfaf5c6c9e0d940908d75dc6af4fd856baa5535c922ee02b1/librt-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8992ca186a1678107b0af3d0c9303d8c7305981b9914989b9788319ed4d89546", size = 235420, upload-time = "2026-02-12T14:53:27.047Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f2/65d86bd462e9c351326564ca805e8457442149f348496e25ccd94583ffa2/librt-0.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:001e5330093d887b8b9165823eca6c5c4db183fe4edea4fdc0680bbac5f46944", size = 246452, upload-time = "2026-02-12T14:53:28.341Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/39c88b503b4cb3fcbdeb3caa29672b6b44ebee8dcc8a54d49839ac280f3f/librt-0.8.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d920789eca7ef71df7f31fd547ec0d3002e04d77f30ba6881e08a630e7b2c30e", size = 238891, upload-time = "2026-02-12T14:53:29.625Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c6/6c0d68190893d01b71b9569b07a1c811e280c0065a791249921c83dc0290/librt-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:82fb4602d1b3e303a58bfe6165992b5a78d823ec646445356c332cd5f5bbaa61", size = 250249, upload-time = "2026-02-12T14:53:30.93Z" }, + { url = "https://files.pythonhosted.org/packages/52/7a/f715ed9e039035d0ea637579c3c0155ab3709a7046bc408c0fb05d337121/librt-0.8.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4d3e38797eb482485b486898f89415a6ab163bc291476bd95712e42cf4383c05", size = 240642, upload-time = "2026-02-12T14:53:32.174Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3c/609000a333debf5992efe087edc6467c1fdbdddca5b610355569bbea9589/librt-0.8.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a905091a13e0884701226860836d0386b88c72ce5c2fdfba6618e14c72be9f25", size = 239621, upload-time = "2026-02-12T14:53:33.39Z" }, + { url = "https://files.pythonhosted.org/packages/b9/df/87b0673d5c395a8f34f38569c116c93142d4dc7e04af2510620772d6bd4f/librt-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:375eda7acfce1f15f5ed56cfc960669eefa1ec8732e3e9087c3c4c3f2066759c", size = 262986, upload-time = "2026-02-12T14:53:34.617Z" }, + { url = "https://files.pythonhosted.org/packages/09/7f/6bbbe9dcda649684773aaea78b87fff4d7e59550fbc2877faa83612087a3/librt-0.8.0-cp314-cp314t-win32.whl", hash = "sha256:2ccdd20d9a72c562ffb73098ac411de351b53a6fbb3390903b2d33078ef90447", size = 51328, upload-time = "2026-02-12T14:53:36.15Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f3/e1981ab6fa9b41be0396648b5850267888a752d025313a9e929c4856208e/librt-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:25e82d920d4d62ad741592fcf8d0f3bda0e3fc388a184cb7d2f566c681c5f7b9", size = 58719, upload-time = "2026-02-12T14:53:37.183Z" }, + { url = "https://files.pythonhosted.org/packages/94/d1/433b3c06e78f23486fe4fdd19bc134657eb30997d2054b0dbf52bbf3382e/librt-0.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:92249938ab744a5890580d3cb2b22042f0dce71cdaa7c1369823df62bedf7cbc", size = 48753, upload-time = "2026-02-12T14:53:38.539Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "moto" +version = "5.1.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "cryptography" }, + { name = "jinja2" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "responses" }, + { name = "werkzeug" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/f8/81e2ee90f47a6ae1e475a961bd6a1a1569b04999ba941897b87101b0d5af/moto-5.1.21.tar.gz", hash = "sha256:713dde46e71e2714fa9a29eec513ec618d35e1d84c256331b5aab3f30692feeb", size = 8441171, upload-time = "2026-02-08T21:52:39.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/c7/4b0bc06f0811caa67f7e8c3ca2e637bd8cb4317c2f8839b7d643d7ace68c/moto-5.1.21-py3-none-any.whl", hash = "sha256:311a30095b08b39dd2707f161f1440d361684fe0090b9fd0751dfd1c9b022445", size = 6514163, upload-time = "2026-02-08T21:52:36.91Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, + { url = "https://files.pythonhosted.org/packages/de/9f/a6abae693f7a0c697dbb435aac52e958dc8da44e92e08ba88d2e42326176/mypy-1.19.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3157c7594ff2ef1634ee058aafc56a82db665c9438fd41b390f3bde1ab12250", size = 13201927, upload-time = "2025-12-15T05:02:29.138Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a4/45c35ccf6e1c65afc23a069f50e2c66f46bd3798cbe0d680c12d12935caa/mypy-1.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdb12f69bcc02700c2b47e070238f42cb87f18c0bc1fc4cdb4fb2bc5fd7a3b8b", size = 12206730, upload-time = "2025-12-15T05:03:01.325Z" }, + { url = "https://files.pythonhosted.org/packages/05/bb/cdcf89678e26b187650512620eec8368fded4cfd99cfcb431e4cdfd19dec/mypy-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f859fb09d9583a985be9a493d5cfc5515b56b08f7447759a0c5deaf68d80506e", size = 12724581, upload-time = "2025-12-15T05:03:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/d1/32/dd260d52babf67bad8e6770f8e1102021877ce0edea106e72df5626bb0ec/mypy-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9a6538e0415310aad77cb94004ca6482330fece18036b5f360b62c45814c4ef", size = 13616252, upload-time = "2025-12-15T05:02:49.036Z" }, + { url = "https://files.pythonhosted.org/packages/71/d0/5e60a9d2e3bd48432ae2b454b7ef2b62a960ab51292b1eda2a95edd78198/mypy-1.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da4869fc5e7f62a88f3fe0b5c919d1d9f7ea3cef92d3689de2823fd27e40aa75", size = 13840848, upload-time = "2025-12-15T05:02:55.95Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/d32051fa65ecf6cc8c6610956473abdc9b4c43301107476ac03559507843/mypy-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:016f2246209095e8eda7538944daa1d60e1e8134d98983b9fc1e92c1fc0cb8dd", size = 10135510, upload-time = "2025-12-15T05:02:58.438Z" }, + { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, + { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "ply" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", size = 159130, upload-time = "2018-02-15T19:01:31.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, +] + +[[package]] +name = "pycparser" +version = "2.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/3a/5031723c09068e9c8c2f0bc25c3a9245f2b1d1aea8396c787a408f2b95ca/pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4", size = 103642, upload-time = "2023-10-25T21:06:56.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e", size = 83537, upload-time = "2023-10-25T21:06:24.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/f1/338d0050b24c3132bcfc79b68c3a5f54bce3d213ecef74d37e988b971d8a/pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e", size = 122615, upload-time = "2023-10-25T21:06:25.815Z" }, + { url = "https://files.pythonhosted.org/packages/07/3a/e56d6431b713518094fae6ff833a04a6f49ad0fbe25fb7c0dc7408e19d20/pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3", size = 122335, upload-time = "2023-10-25T21:06:28.631Z" }, + { url = "https://files.pythonhosted.org/packages/4a/bb/5f40a4d5e985a43b43f607250e766cdec28904682c3505eb0bd343a4b7db/pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d", size = 118510, upload-time = "2023-10-25T21:06:30.718Z" }, + { url = "https://files.pythonhosted.org/packages/1c/13/e6a22f40f5800af116c02c28e29f15c06aa41cb2036f6a64ab124647f28b/pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174", size = 60865, upload-time = "2023-10-25T21:06:32.742Z" }, + { url = "https://files.pythonhosted.org/packages/75/ef/2fa3b55023ec07c22682c957808f9a41836da4cd006b5f55ec76bf0fbfa6/pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d", size = 63239, upload-time = "2023-10-25T21:06:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/23/88/0acd180010aaed4987c85700b7cc17f9505f3edb4e5873e4dc67f613e338/pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b", size = 58106, upload-time = "2023-10-25T21:06:54.387Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.32.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/ec/535bf6f9bd280de6a4637526602a146a68fde757100ecf8c9333173392db/requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289", size = 130327, upload-time = "2024-05-21T18:51:32.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/20/748e38b466e0819491f0ce6e90ebe4184966ee304fe483e2c414b0f4ef07/requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c", size = 63902, upload-time = "2024-05-21T18:51:29.562Z" }, +] + +[[package]] +name = "responses" +version = "0.25.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/95/89c054ad70bfef6da605338b009b2e283485835351a9935c7bfbfaca7ffc/responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4", size = 79320, upload-time = "2025-08-08T19:01:46.709Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/4c/cc276ce57e572c102d9542d383b2cfd551276581dc60004cb94fe8774c11/responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c", size = 34769, upload-time = "2025-08-08T19:01:45.018Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/06/f71e3a86b2df0dfa2d2f72195941cd09b44f87711cb7fa5193732cb9a5fc/ruff-0.14.14.tar.gz", hash = "sha256:2d0f819c9a90205f3a867dbbd0be083bee9912e170fd7d9704cc8ae45824896b", size = 4515732, upload-time = "2026-01-22T22:30:17.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/89/20a12e97bc6b9f9f68343952da08a8099c57237aef953a56b82711d55edd/ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed", size = 10467650, upload-time = "2026-01-22T22:30:08.578Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b1/c5de3fd2d5a831fcae21beda5e3589c0ba67eec8202e992388e4b17a6040/ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c", size = 10883245, upload-time = "2026-01-22T22:30:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7c/3c1db59a10e7490f8f6f8559d1db8636cbb13dccebf18686f4e3c9d7c772/ruff-0.14.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:026c1d25996818f0bf498636686199d9bd0d9d6341c9c2c3b62e2a0198b758de", size = 10231273, upload-time = "2026-01-22T22:30:34.642Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6e/5e0e0d9674be0f8581d1f5e0f0a04761203affce3232c1a1189d0e3b4dad/ruff-0.14.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e", size = 10585753, upload-time = "2026-01-22T22:30:31.781Z" }, + { url = "https://files.pythonhosted.org/packages/23/09/754ab09f46ff1884d422dc26d59ba18b4e5d355be147721bb2518aa2a014/ruff-0.14.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c0f18b922c6d2ff9a5e6c3ee16259adc513ca775bcf82c67ebab7cbd9da5bc8", size = 10286052, upload-time = "2026-01-22T22:30:24.827Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cc/e71f88dd2a12afb5f50733851729d6b571a7c3a35bfdb16c3035132675a0/ruff-0.14.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1629e67489c2dea43e8658c3dba659edbfd87361624b4040d1df04c9740ae906", size = 11043637, upload-time = "2026-01-22T22:30:13.239Z" }, + { url = "https://files.pythonhosted.org/packages/67/b2/397245026352494497dac935d7f00f1468c03a23a0c5db6ad8fc49ca3fb2/ruff-0.14.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:27493a2131ea0f899057d49d303e4292b2cae2bb57253c1ed1f256fbcd1da480", size = 12194761, upload-time = "2026-01-22T22:30:22.542Z" }, + { url = "https://files.pythonhosted.org/packages/5b/06/06ef271459f778323112c51b7587ce85230785cd64e91772034ddb88f200/ruff-0.14.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01ff589aab3f5b539e35db38425da31a57521efd1e4ad1ae08fc34dbe30bd7df", size = 12005701, upload-time = "2026-01-22T22:30:20.499Z" }, + { url = "https://files.pythonhosted.org/packages/41/d6/99364514541cf811ccc5ac44362f88df66373e9fec1b9d1c4cc830593fe7/ruff-0.14.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc12d74eef0f29f51775f5b755913eb523546b88e2d733e1d701fe65144e89b", size = 11282455, upload-time = "2026-01-22T22:29:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/ca/71/37daa46f89475f8582b7762ecd2722492df26421714a33e72ccc9a84d7a5/ruff-0.14.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb8481604b7a9e75eff53772496201690ce2687067e038b3cc31aaf16aa0b974", size = 11215882, upload-time = "2026-01-22T22:29:57.032Z" }, + { url = "https://files.pythonhosted.org/packages/2c/10/a31f86169ec91c0705e618443ee74ede0bdd94da0a57b28e72db68b2dbac/ruff-0.14.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:14649acb1cf7b5d2d283ebd2f58d56b75836ed8c6f329664fa91cdea19e76e66", size = 11180549, upload-time = "2026-01-22T22:30:27.175Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1e/c723f20536b5163adf79bdd10c5f093414293cdf567eed9bdb7b83940f3f/ruff-0.14.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13", size = 10543416, upload-time = "2026-01-22T22:30:01.964Z" }, + { url = "https://files.pythonhosted.org/packages/3e/34/8a84cea7e42c2d94ba5bde1d7a4fae164d6318f13f933d92da6d7c2041ff/ruff-0.14.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412", size = 10285491, upload-time = "2026-01-22T22:30:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/55/ef/b7c5ea0be82518906c978e365e56a77f8de7678c8bb6651ccfbdc178c29f/ruff-0.14.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:cc8b22da8d9d6fdd844a68ae937e2a0adf9b16514e9a97cc60355e2d4b219fc3", size = 10733525, upload-time = "2026-01-22T22:30:06.499Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/aaf1dfbcc53a2811f6cc0a1759de24e4b03e02ba8762daabd9b6bd8c59e3/ruff-0.14.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:16bc890fb4cc9781bb05beb5ab4cd51be9e7cb376bf1dd3580512b24eb3fda2b", size = 11315626, upload-time = "2026-01-22T22:30:36.848Z" }, + { url = "https://files.pythonhosted.org/packages/2c/aa/9f89c719c467dfaf8ad799b9bae0df494513fb21d31a6059cb5870e57e74/ruff-0.14.14-py3-none-win32.whl", hash = "sha256:b530c191970b143375b6a68e6f743800b2b786bbcf03a7965b06c4bf04568167", size = 10502442, upload-time = "2026-01-22T22:30:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/87/44/90fa543014c45560cae1fffc63ea059fb3575ee6e1cb654562197e5d16fb/ruff-0.14.14-py3-none-win_amd64.whl", hash = "sha256:3dde1435e6b6fe5b66506c1dff67a421d0b7f6488d466f651c07f4cab3bf20fd", size = 11630486, upload-time = "2026-01-22T22:30:10.852Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/70/1469ef1d3542ae7c2c7b72bd5e3a4e6ee69d7978fa8a3af05a38eca5becf/werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67", size = 864754, upload-time = "2026-01-08T17:49:23.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/e4/8d97cca767bcc1be76d16fb76951608305561c6e056811587f36cb1316a8/werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", size = 225025, upload-time = "2026-01-08T17:49:21.859Z" }, +] + +[[package]] +name = "xmltodict" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/aa/917ceeed4dbb80d2f04dbd0c784b7ee7bba8ae5a54837ef0e5e062cd3cfb/xmltodict-1.0.2.tar.gz", hash = "sha256:54306780b7c2175a3967cad1db92f218207e5bc1aba697d887807c0fb68b7649", size = 25725, upload-time = "2025-09-17T21:59:26.459Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/20/69a0e6058bc5ea74892d089d64dfc3a62ba78917ec5e2cfa70f7c92ba3a5/xmltodict-1.0.2-py3-none-any.whl", hash = "sha256:62d0fddb0dcbc9f642745d8bbf4d81fd17d6dfaec5a15b5c1876300aad92af0d", size = 13893, upload-time = "2025-09-17T21:59:24.859Z" }, +] diff --git a/tasks/granule-invalidator/pyproject.toml b/tasks/granule-invalidator/pyproject.toml index 56919efc2f49..1c25fd098e1e 100644 --- a/tasks/granule-invalidator/pyproject.toml +++ b/tasks/granule-invalidator/pyproject.toml @@ -17,8 +17,11 @@ repository = "https://github.com/nasa/cumulus.git" [dependency-groups] dev = [ + "mypy>=1.0.0", + "ruff~=0.14.0", "pytest>=9.0.2", "pytest-cov>=7.0.0", + "pydantic>=2.10.4", ] [tool.uv.sources] diff --git a/tasks/granule-invalidator/uv.lock b/tasks/granule-invalidator/uv.lock index 22b3db15e6fa..20a91c7fb41d 100644 --- a/tasks/granule-invalidator/uv.lock +++ b/tasks/granule-invalidator/uv.lock @@ -2,6 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.12" +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + [[package]] name = "attrs" version = "25.4.0" @@ -319,8 +328,11 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "mypy" }, + { name = "pydantic" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "ruff" }, ] [package.metadata] @@ -332,8 +344,11 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "mypy", specifier = ">=1.0.0" }, + { name = "pydantic", specifier = ">=2.10.4" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "ruff", specifier = "~=0.14.0" }, ] [[package]] @@ -388,6 +403,108 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/97/c698bd9350f307daad79dd740806e1a59becd693bd11443a0f531e3229b3/jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6", size = 90379, upload-time = "2022-11-29T20:37:45.842Z" }, ] +[[package]] +name = "librt" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", size = 177471, upload-time = "2026-02-17T16:13:06.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/21/d39b0a87ac52fc98f621fb6f8060efb017a767ebbbac2f99fbcbc9ddc0d7/librt-0.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a28f2612ab566b17f3698b0da021ff9960610301607c9a5e8eaca62f5e1c350a", size = 66516, upload-time = "2026-02-17T16:11:41.604Z" }, + { url = "https://files.pythonhosted.org/packages/69/f1/46375e71441c43e8ae335905e069f1c54febee63a146278bcee8782c84fd/librt-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:60a78b694c9aee2a0f1aaeaa7d101cf713e92e8423a941d2897f4fa37908dab9", size = 68634, upload-time = "2026-02-17T16:11:43.268Z" }, + { url = "https://files.pythonhosted.org/packages/0a/33/c510de7f93bf1fa19e13423a606d8189a02624a800710f6e6a0a0f0784b3/librt-0.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:758509ea3f1eba2a57558e7e98f4659d0ea7670bff49673b0dde18a3c7e6c0eb", size = 198941, upload-time = "2026-02-17T16:11:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/dd/36/e725903416409a533d92398e88ce665476f275081d0d7d42f9c4951999e5/librt-0.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:039b9f2c506bd0ab0f8725aa5ba339c6f0cd19d3b514b50d134789809c24285d", size = 209991, upload-time = "2026-02-17T16:11:45.462Z" }, + { url = "https://files.pythonhosted.org/packages/30/7a/8d908a152e1875c9f8eac96c97a480df425e657cdb47854b9efaa4998889/librt-0.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bb54f1205a3a6ab41a6fd71dfcdcbd278670d3a90ca502a30d9da583105b6f7", size = 224476, upload-time = "2026-02-17T16:11:46.542Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/a22c34f2c485b8903a06f3fe3315341fe6876ef3599792344669db98fcff/librt-0.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:05bd41cdee35b0c59c259f870f6da532a2c5ca57db95b5f23689fcb5c9e42440", size = 217518, upload-time = "2026-02-17T16:11:47.746Z" }, + { url = "https://files.pythonhosted.org/packages/79/6f/5c6fea00357e4f82ba44f81dbfb027921f1ab10e320d4a64e1c408d035d9/librt-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adfab487facf03f0d0857b8710cf82d0704a309d8ffc33b03d9302b4c64e91a9", size = 225116, upload-time = "2026-02-17T16:11:49.298Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a0/95ced4e7b1267fe1e2720a111685bcddf0e781f7e9e0ce59d751c44dcfe5/librt-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:153188fe98a72f206042be10a2c6026139852805215ed9539186312d50a8e972", size = 217751, upload-time = "2026-02-17T16:11:50.49Z" }, + { url = "https://files.pythonhosted.org/packages/93/c2/0517281cb4d4101c27ab59472924e67f55e375bc46bedae94ac6dc6e1902/librt-0.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:dd3c41254ee98604b08bd5b3af5bf0a89740d4ee0711de95b65166bf44091921", size = 218378, upload-time = "2026-02-17T16:11:51.783Z" }, + { url = "https://files.pythonhosted.org/packages/43/e8/37b3ac108e8976888e559a7b227d0ceac03c384cfd3e7a1c2ee248dbae79/librt-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e0d138c7ae532908cbb342162b2611dbd4d90c941cd25ab82084aaf71d2c0bd0", size = 241199, upload-time = "2026-02-17T16:11:53.561Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/35812d041c53967fedf551a39399271bbe4257e681236a2cf1a69c8e7fa1/librt-0.8.1-cp312-cp312-win32.whl", hash = "sha256:43353b943613c5d9c49a25aaffdba46f888ec354e71e3529a00cca3f04d66a7a", size = 54917, upload-time = "2026-02-17T16:11:54.758Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/fa5d5331b862b9775aaf2a100f5ef86854e5d4407f71bddf102f4421e034/librt-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:ff8baf1f8d3f4b6b7257fcb75a501f2a5499d0dda57645baa09d4d0d34b19444", size = 62017, upload-time = "2026-02-17T16:11:55.748Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7c/c614252f9acda59b01a66e2ddfd243ed1c7e1deab0293332dfbccf862808/librt-0.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:0f2ae3725904f7377e11cc37722d5d401e8b3d5851fb9273d7f4fe04f6b3d37d", size = 52441, upload-time = "2026-02-17T16:11:56.801Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3c/f614c8e4eaac7cbf2bbdf9528790b21d89e277ee20d57dc6e559c626105f/librt-0.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e6bad1cd94f6764e1e21950542f818a09316645337fd5ab9a7acc45d99a8f35", size = 66529, upload-time = "2026-02-17T16:11:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/ab/96/5836544a45100ae411eda07d29e3d99448e5258b6e9c8059deb92945f5c2/librt-0.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf450f498c30af55551ba4f66b9123b7185362ec8b625a773b3d39aa1a717583", size = 68669, upload-time = "2026-02-17T16:11:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/06/53/f0b992b57af6d5531bf4677d75c44f095f2366a1741fb695ee462ae04b05/librt-0.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eca45e982fa074090057132e30585a7e8674e9e885d402eae85633e9f449ce6c", size = 199279, upload-time = "2026-02-17T16:11:59.862Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/4848cc16e268d14280d8168aee4f31cea92bbd2b79ce33d3e166f2b4e4fc/librt-0.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c3811485fccfda840861905b8c70bba5ec094e02825598bb9d4ca3936857a04", size = 210288, upload-time = "2026-02-17T16:12:00.954Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/27fdc2e95de26273d83b96742d8d3b7345f2ea2bdbd2405cc504644f2096/librt-0.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4af413908f77294605e28cfd98063f54b2c790561383971d2f52d113d9c363", size = 224809, upload-time = "2026-02-17T16:12:02.108Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d0/78200a45ba3240cb042bc597d6f2accba9193a2c57d0356268cbbe2d0925/librt-0.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5212a5bd7fae98dae95710032902edcd2ec4dc994e883294f75c857b83f9aba0", size = 218075, upload-time = "2026-02-17T16:12:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/a210839fa74c90474897124c064ffca07f8d4b347b6574d309686aae7ca6/librt-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e692aa2d1d604e6ca12d35e51fdc36f4cda6345e28e36374579f7ef3611b3012", size = 225486, upload-time = "2026-02-17T16:12:04.725Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c1/a03cc63722339ddbf087485f253493e2b013039f5b707e8e6016141130fa/librt-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4be2a5c926b9770c9e08e717f05737a269b9d0ebc5d2f0060f0fe3fe9ce47acb", size = 218219, upload-time = "2026-02-17T16:12:05.828Z" }, + { url = "https://files.pythonhosted.org/packages/58/f5/fff6108af0acf941c6f274a946aea0e484bd10cd2dc37610287ce49388c5/librt-0.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fd1a720332ea335ceb544cf0a03f81df92abd4bb887679fd1e460976b0e6214b", size = 218750, upload-time = "2026-02-17T16:12:07.09Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/5a387bfef30ec1e4b4f30562c8586566faf87e47d696768c19feb49e3646/librt-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2af9e01e0ef80d95ae3c720be101227edae5f2fe7e3dc63d8857fadfc5a1d", size = 241624, upload-time = "2026-02-17T16:12:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/24f8502db11d405232ac1162eb98069ca49c3306c1d75c6ccc61d9af8789/librt-0.8.1-cp313-cp313-win32.whl", hash = "sha256:086a32dbb71336627e78cc1d6ee305a68d038ef7d4c39aaff41ae8c9aa46e91a", size = 54969, upload-time = "2026-02-17T16:12:09.633Z" }, + { url = "https://files.pythonhosted.org/packages/5c/73/c9fdf6cb2a529c1a092ce769a12d88c8cca991194dfe641b6af12fa964d2/librt-0.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:e11769a1dbda4da7b00a76cfffa67aa47cfa66921d2724539eee4b9ede780b79", size = 62000, upload-time = "2026-02-17T16:12:10.632Z" }, + { url = "https://files.pythonhosted.org/packages/d3/97/68f80ca3ac4924f250cdfa6e20142a803e5e50fca96ef5148c52ee8c10ea/librt-0.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:924817ab3141aca17893386ee13261f1d100d1ef410d70afe4389f2359fea4f0", size = 52495, upload-time = "2026-02-17T16:12:11.633Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", size = 66081, upload-time = "2026-02-17T16:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", size = 68309, upload-time = "2026-02-17T16:12:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", size = 196804, upload-time = "2026-02-17T16:12:14.818Z" }, + { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", size = 206907, upload-time = "2026-02-17T16:12:16.513Z" }, + { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", size = 221217, upload-time = "2026-02-17T16:12:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", size = 214622, upload-time = "2026-02-17T16:12:19.108Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", size = 221987, upload-time = "2026-02-17T16:12:20.331Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", size = 215132, upload-time = "2026-02-17T16:12:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", size = 215195, upload-time = "2026-02-17T16:12:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", size = 237946, upload-time = "2026-02-17T16:12:24.275Z" }, + { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", size = 50689, upload-time = "2026-02-17T16:12:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", size = 57875, upload-time = "2026-02-17T16:12:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", size = 48058, upload-time = "2026-02-17T16:12:28.556Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", size = 68313, upload-time = "2026-02-17T16:12:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", size = 70994, upload-time = "2026-02-17T16:12:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", size = 220770, upload-time = "2026-02-17T16:12:33.294Z" }, + { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", size = 235409, upload-time = "2026-02-17T16:12:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", size = 246473, upload-time = "2026-02-17T16:12:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", size = 238866, upload-time = "2026-02-17T16:12:37.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", size = 250248, upload-time = "2026-02-17T16:12:39.445Z" }, + { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", size = 240629, upload-time = "2026-02-17T16:12:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", size = 239615, upload-time = "2026-02-17T16:12:42.446Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", size = 263001, upload-time = "2026-02-17T16:12:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", size = 51328, upload-time = "2026-02-17T16:12:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", size = 58722, upload-time = "2026-02-17T16:12:46.85Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, + { url = "https://files.pythonhosted.org/packages/de/9f/a6abae693f7a0c697dbb435aac52e958dc8da44e92e08ba88d2e42326176/mypy-1.19.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3157c7594ff2ef1634ee058aafc56a82db665c9438fd41b390f3bde1ab12250", size = 13201927, upload-time = "2025-12-15T05:02:29.138Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a4/45c35ccf6e1c65afc23a069f50e2c66f46bd3798cbe0d680c12d12935caa/mypy-1.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdb12f69bcc02700c2b47e070238f42cb87f18c0bc1fc4cdb4fb2bc5fd7a3b8b", size = 12206730, upload-time = "2025-12-15T05:03:01.325Z" }, + { url = "https://files.pythonhosted.org/packages/05/bb/cdcf89678e26b187650512620eec8368fded4cfd99cfcb431e4cdfd19dec/mypy-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f859fb09d9583a985be9a493d5cfc5515b56b08f7447759a0c5deaf68d80506e", size = 12724581, upload-time = "2025-12-15T05:03:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/d1/32/dd260d52babf67bad8e6770f8e1102021877ce0edea106e72df5626bb0ec/mypy-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9a6538e0415310aad77cb94004ca6482330fece18036b5f360b62c45814c4ef", size = 13616252, upload-time = "2025-12-15T05:02:49.036Z" }, + { url = "https://files.pythonhosted.org/packages/71/d0/5e60a9d2e3bd48432ae2b454b7ef2b62a960ab51292b1eda2a95edd78198/mypy-1.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da4869fc5e7f62a88f3fe0b5c919d1d9f7ea3cef92d3689de2823fd27e40aa75", size = 13840848, upload-time = "2025-12-15T05:02:55.95Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/d32051fa65ecf6cc8c6610956473abdc9b4c43301107476ac03559507843/mypy-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:016f2246209095e8eda7538944daa1d60e1e8134d98983b9fc1e92c1fc0cb8dd", size = 10135510, upload-time = "2025-12-15T05:02:58.438Z" }, + { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, + { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -397,6 +514,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -424,6 +550,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, ] +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -530,6 +742,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] +[[package]] +name = "ruff" +version = "0.14.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/06/f71e3a86b2df0dfa2d2f72195941cd09b44f87711cb7fa5193732cb9a5fc/ruff-0.14.14.tar.gz", hash = "sha256:2d0f819c9a90205f3a867dbbd0be083bee9912e170fd7d9704cc8ae45824896b", size = 4515732, upload-time = "2026-01-22T22:30:17.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/89/20a12e97bc6b9f9f68343952da08a8099c57237aef953a56b82711d55edd/ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed", size = 10467650, upload-time = "2026-01-22T22:30:08.578Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b1/c5de3fd2d5a831fcae21beda5e3589c0ba67eec8202e992388e4b17a6040/ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c", size = 10883245, upload-time = "2026-01-22T22:30:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7c/3c1db59a10e7490f8f6f8559d1db8636cbb13dccebf18686f4e3c9d7c772/ruff-0.14.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:026c1d25996818f0bf498636686199d9bd0d9d6341c9c2c3b62e2a0198b758de", size = 10231273, upload-time = "2026-01-22T22:30:34.642Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6e/5e0e0d9674be0f8581d1f5e0f0a04761203affce3232c1a1189d0e3b4dad/ruff-0.14.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e", size = 10585753, upload-time = "2026-01-22T22:30:31.781Z" }, + { url = "https://files.pythonhosted.org/packages/23/09/754ab09f46ff1884d422dc26d59ba18b4e5d355be147721bb2518aa2a014/ruff-0.14.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c0f18b922c6d2ff9a5e6c3ee16259adc513ca775bcf82c67ebab7cbd9da5bc8", size = 10286052, upload-time = "2026-01-22T22:30:24.827Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cc/e71f88dd2a12afb5f50733851729d6b571a7c3a35bfdb16c3035132675a0/ruff-0.14.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1629e67489c2dea43e8658c3dba659edbfd87361624b4040d1df04c9740ae906", size = 11043637, upload-time = "2026-01-22T22:30:13.239Z" }, + { url = "https://files.pythonhosted.org/packages/67/b2/397245026352494497dac935d7f00f1468c03a23a0c5db6ad8fc49ca3fb2/ruff-0.14.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:27493a2131ea0f899057d49d303e4292b2cae2bb57253c1ed1f256fbcd1da480", size = 12194761, upload-time = "2026-01-22T22:30:22.542Z" }, + { url = "https://files.pythonhosted.org/packages/5b/06/06ef271459f778323112c51b7587ce85230785cd64e91772034ddb88f200/ruff-0.14.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01ff589aab3f5b539e35db38425da31a57521efd1e4ad1ae08fc34dbe30bd7df", size = 12005701, upload-time = "2026-01-22T22:30:20.499Z" }, + { url = "https://files.pythonhosted.org/packages/41/d6/99364514541cf811ccc5ac44362f88df66373e9fec1b9d1c4cc830593fe7/ruff-0.14.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc12d74eef0f29f51775f5b755913eb523546b88e2d733e1d701fe65144e89b", size = 11282455, upload-time = "2026-01-22T22:29:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/ca/71/37daa46f89475f8582b7762ecd2722492df26421714a33e72ccc9a84d7a5/ruff-0.14.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb8481604b7a9e75eff53772496201690ce2687067e038b3cc31aaf16aa0b974", size = 11215882, upload-time = "2026-01-22T22:29:57.032Z" }, + { url = "https://files.pythonhosted.org/packages/2c/10/a31f86169ec91c0705e618443ee74ede0bdd94da0a57b28e72db68b2dbac/ruff-0.14.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:14649acb1cf7b5d2d283ebd2f58d56b75836ed8c6f329664fa91cdea19e76e66", size = 11180549, upload-time = "2026-01-22T22:30:27.175Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1e/c723f20536b5163adf79bdd10c5f093414293cdf567eed9bdb7b83940f3f/ruff-0.14.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13", size = 10543416, upload-time = "2026-01-22T22:30:01.964Z" }, + { url = "https://files.pythonhosted.org/packages/3e/34/8a84cea7e42c2d94ba5bde1d7a4fae164d6318f13f933d92da6d7c2041ff/ruff-0.14.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412", size = 10285491, upload-time = "2026-01-22T22:30:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/55/ef/b7c5ea0be82518906c978e365e56a77f8de7678c8bb6651ccfbdc178c29f/ruff-0.14.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:cc8b22da8d9d6fdd844a68ae937e2a0adf9b16514e9a97cc60355e2d4b219fc3", size = 10733525, upload-time = "2026-01-22T22:30:06.499Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/aaf1dfbcc53a2811f6cc0a1759de24e4b03e02ba8762daabd9b6bd8c59e3/ruff-0.14.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:16bc890fb4cc9781bb05beb5ab4cd51be9e7cb376bf1dd3580512b24eb3fda2b", size = 11315626, upload-time = "2026-01-22T22:30:36.848Z" }, + { url = "https://files.pythonhosted.org/packages/2c/aa/9f89c719c467dfaf8ad799b9bae0df494513fb21d31a6059cb5870e57e74/ruff-0.14.14-py3-none-win32.whl", hash = "sha256:b530c191970b143375b6a68e6f743800b2b786bbcf03a7965b06c4bf04568167", size = 10502442, upload-time = "2026-01-22T22:30:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/87/44/90fa543014c45560cae1fffc63ea059fb3575ee6e1cb654562197e5d16fb/ruff-0.14.14-py3-none-win_amd64.whl", hash = "sha256:3dde1435e6b6fe5b66506c1dff67a421d0b7f6488d466f651c07f4cab3bf20fd", size = 11630486, upload-time = "2026-01-22T22:30:10.852Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, +] + [[package]] name = "s3transfer" version = "0.14.0" @@ -560,6 +798,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + [[package]] name = "urllib3" version = "2.6.3" From 320b3fa860f1ae37fb8b923313e739ddb8b01e78 Mon Sep 17 00:00:00 2001 From: Brandon Lokey <57154350+brandonlokey@users.noreply.github.com> Date: Tue, 24 Feb 2026 09:46:57 -0500 Subject: [PATCH 15/26] CUMULUS-4352: implemented multipart downloads in AddMissingFileChecksum core task (#4211) * implemented multipart checksum computation based on threshold and partition values * fixed ESLint errors and warnings * add retries to s3 getObject calls using aws-client, cleaned up unit tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleaned up granule size variables and arguments * cleaned up typing, removed env variables for setting retries * updated CHANGELOG with CUMULUS-4352 changes * updated unit tests to check updateHashFromBody branches --------- Co-authored-by: Brandon Lokey Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 + tasks/add-missing-file-checksums/src/index.ts | 89 ++++++++++++++++++- tasks/add-missing-file-checksums/src/types.ts | 1 + .../tests/test-index.ts | 78 +++++++++++++++- 4 files changed, 164 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96555f2e1c06..5a656321c351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added supporting Terraform for the CnmResponse task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus. - **CUMULUS-4498** - Added `states:StartExecution` action to the `-steprole` IAM role. +- **CUMULUS-4352** + - Implemented multi-part download support for checksum computation in addMissingFileChecksums task. - **CUMULUS-4542** - Created the `aws-api-proxy` coreified task, which provides the functionality to post a list of CNM messages to a specified SNS topic. - **CUMULUS-4517** diff --git a/tasks/add-missing-file-checksums/src/index.ts b/tasks/add-missing-file-checksums/src/index.ts index 37f0beb550a9..e32b26ab2a85 100644 --- a/tasks/add-missing-file-checksums/src/index.ts +++ b/tasks/add-missing-file-checksums/src/index.ts @@ -3,16 +3,95 @@ import * as awsClients from '@cumulus/aws-client/services'; import * as S3 from '@cumulus/aws-client/S3'; import { Context } from 'aws-lambda'; import { CumulusMessage, CumulusRemoteMessage } from '@cumulus/types/message'; +import crypto from 'crypto'; +import type { Readable } from 'stream'; import { Granule, GranuleFile, HandlerInput, HandlerEvent } from './types'; +export const updateHashFromBody = async (hash: crypto.Hash, + body: Readable | Buffer | Uint8Array | undefined) => { + if (!body) return; + + // Node readable stream + if (typeof (body as Readable).on === 'function') { + const stream = body as Readable; + await new Promise((resolve, reject) => { + stream.on('data', (chunk: any) => hash.update(chunk as any)); + stream.on('end', resolve); + stream.on('error', reject); + }); + return; + } + + // Normalize into buffer and set to Uint8Array if not a stream + const buf = Buffer.isBuffer(body) ? body : Buffer.from(body); + hash.update(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)); +}; + +const calculateObjectHashByRanges = async (params: { + s3: { getObject: S3.GetObjectMethod } + algorithm: string, + bucket: string, + key: string, + size: number, + partSizeBytes: number +}) => { + const { + s3, algorithm, bucket, key, size, partSizeBytes, + } = params; + + const hash = crypto.createHash(algorithm); + + const ranges: { start: number; end: number }[] = []; + + for (let start = 0; start < size; start += partSizeBytes) { + const end = Math.min(start + partSizeBytes - 1, size - 1); + ranges.push({ start, end }); + } + + await ranges.reduce>( + (p, { start, end }) => + p.then(async () => { + const resp = await s3.getObject({ + Bucket: bucket, + Key: key, + Range: `bytes=${start}-${end}`, + }); + + return updateHashFromBody(hash, (resp as any).Body); + }), + Promise.resolve() + ); + + return hash.digest('hex'); +}; + const calculateGranuleFileChecksum = async (params: { s3: { getObject: S3.GetObjectMethod }, algorithm: string, granuleFile: GranuleFile }) => { const { s3, algorithm, granuleFile } = params; - - const { bucket, key } = granuleFile; + const { bucket, key, size } = granuleFile; + + const thresholdMb = Number(process.env.MULTIPART_CHECKSUM_THRESHOLD_MEGABYTES || 0); + const partMb = Number(process.env.MULTIPART_CHECKSUM_PART_MEGABYTES || 0); + const thresholdBytes = thresholdMb * 1024 * 1024; + const partSizeBytes = partMb * 1024 * 1024; + + const partitioningEnabled = (thresholdMb > 0 && partMb > 0) + && (size > thresholdBytes); + + // Calculate checksum by partitioning + if (partitioningEnabled) { + return await calculateObjectHashByRanges({ + s3, + algorithm, + bucket, + key, + size, + partSizeBytes: partSizeBytes, + }); + } return await S3.calculateObjectHash({ s3, algorithm, bucket, key }); }; @@ -44,7 +123,7 @@ export const addChecksumToGranuleFile = async (params: { } const checksum = await calculateGranuleFileChecksum({ - s3, + s3: s3, algorithm, granuleFile, }); @@ -81,10 +160,12 @@ const addFileChecksumsToGranule = async (params: { export const handler = async (event: HandlerEvent) => { const { config, input } = event; + + const s3Client = awsClients.s3(); const granulesWithChecksums = await Promise.all( input.granules.map( (granule) => addFileChecksumsToGranule({ - s3: awsClients.s3(), + s3: { getObject: (params: any) => S3.getObject(s3Client, params) }, algorithm: config.algorithm, granule, }) diff --git a/tasks/add-missing-file-checksums/src/types.ts b/tasks/add-missing-file-checksums/src/types.ts index 45fa2ee04c39..2d782be4c820 100644 --- a/tasks/add-missing-file-checksums/src/types.ts +++ b/tasks/add-missing-file-checksums/src/types.ts @@ -3,6 +3,7 @@ export interface GranuleFile { checksum?: string, bucket: string, key: string, + size: number, [key: string]: unknown } diff --git a/tasks/add-missing-file-checksums/tests/test-index.ts b/tasks/add-missing-file-checksums/tests/test-index.ts index d2985b01a8c5..7c37e8e3efb8 100644 --- a/tasks/add-missing-file-checksums/tests/test-index.ts +++ b/tasks/add-missing-file-checksums/tests/test-index.ts @@ -3,7 +3,9 @@ import anyTest, { TestFn } from 'ava'; import { Readable } from 'stream'; import * as S3 from '@cumulus/aws-client/S3'; import cryptoRandomString from 'crypto-random-string'; -import { handler, addChecksumToGranuleFile } from '../src'; +import * as crypto from 'crypto'; + +import { handler, addChecksumToGranuleFile, updateHashFromBody } from '../src'; const randomString = () => cryptoRandomString({ length: 10 }); @@ -19,11 +21,22 @@ test.before((t) => { }; }); +const makeRangeS3Mock = (full: Buffer) => ({ + getObject: sinon.fake(async ({ Range }: any) => { + const [, startStr, endStr] = /bytes=(\d+)-(\d+)/.exec(Range) ?? []; + const start = Number(startStr); + const end = Number(endStr); + + return { Body: Readable.from([full.subarray(start, end + 1)]) }; + }), +}); + test('addChecksumToGranuleFile() does not update a granule file if checksumType is set but checksum is not', async (t) => { const granuleFile = { bucket: 'bucket', key: 'key', checksumType: 'md5', + size: 123, }; const result = await addChecksumToGranuleFile({ @@ -40,6 +53,7 @@ test('addChecksumToGranuleFile() does not update a granule file if checksum is s bucket: 'bucket', key: 'key', checksum: 'asdf', + size: 123, }; const result = await addChecksumToGranuleFile({ @@ -70,6 +84,7 @@ test('addChecksumToGranuleFile() returns the file if checksumType and checksum a key: 'key', checksumType: 'md5', checksum: 'asdf', + size: 123, }; const result = await addChecksumToGranuleFile({ @@ -85,6 +100,7 @@ test('addChecksumToGranuleFile() adds the checksumType and checksum to the file const granuleFile = { bucket: 'bucket', key: 'path/to/file.txt', + size: 123, }; const fakeGetObject = sinon.fake.resolves({ @@ -129,6 +145,7 @@ test('The handler does not update files that already have a checksum', async (t) key: 'key', checksumType: 'c-type', checksum: 'c-value', + size: 123, }, ], }, @@ -156,6 +173,8 @@ test('The handler updates files that do not have a checksum', async (t) => { Body: 'asdf', }); + const size = Buffer.byteLength('asdf'); + const event = { config: { algorithm: 'md5', @@ -165,7 +184,7 @@ test('The handler updates files that do not have a checksum', async (t) => { { granuleId: 'g-1', files: [ - { bucket, key }, + { bucket, key, size }, ], }, ], @@ -194,6 +213,7 @@ test('The handler preserves extra input properties', async (t) => { key: 'key', checksumType: 'c-type', checksum: 'c-value', + size: 123, }, ], }, @@ -222,6 +242,7 @@ test('The handler preserves extra granule properties', async (t) => { key: 'key', checksumType: 'c-type', checksum: 'c-value', + size: 123, }, ], }, @@ -250,6 +271,7 @@ test('The handler preserves extra granule file properties', async (t) => { key: 'key', checksumType: 'c-type', checksum: 'c-value', + size: 123, }, ], }, @@ -261,3 +283,55 @@ test('The handler preserves extra granule file properties', async (t) => { t.is(result.granules[0].files[0].foo, 'bar'); }); + +test('addChecksumToGranuleFile() uses ranged GETs for large files and computes correct checksum', async (t) => { + process.env.MULTIPART_CHECKSUM_THRESHOLD_MEGABYTES = '2'; // anything > 2 MB triggers multipart + process.env.MULTIPART_CHECKSUM_PART_MEGABYTES = '1'; // 1 MB ranges + + // Ensure multiple ranges (4 MB) + const full = Buffer.alloc(4 * 1024 * 1024, 'a'); + const s3 = makeRangeS3Mock(full); + + const result = await addChecksumToGranuleFile({ + s3: s3 as any, + algorithm: 'md5', + granuleFile: { + bucket: 'bucket', + key: 'key', + size: full.length, + } as any, + }); + + // It should have called getObject multiple times with Range + const rangedCalls = (s3.getObject as any).getCalls().filter((c: any) => c.args[0].Range); + + // Verify ranges look correct + t.true(rangedCalls.length > 1); + t.is(result.checksumType, 'md5'); + t.is(result.checksum, crypto.createHash('md5').update(new Uint8Array(full.buffer, full.byteOffset, full.byteLength)).digest('hex')); +}); + +test('updateHashFromBody() stream branch', async (t) => { + const hash = crypto.createHash('md5'); + await updateHashFromBody(hash, Readable.from(['test']) as any); + t.is(hash.digest('hex'), crypto.createHash('md5').update('test').digest('hex')); +}); + +test('updateHashFromBody() Buffer branch', async (t) => { + const hash = crypto.createHash('md5'); + const body = Buffer.from('test'); + await updateHashFromBody(hash, Buffer.from('test')); + t.is(hash.digest('hex'), crypto.createHash('md5').update(new Uint8Array(body.buffer, body.byteOffset, body.byteLength)).digest('hex')); +}); + +test('updateHashFromBody() empty body branch', async (t) => { + const hash = crypto.createHash('md5'); + await updateHashFromBody(hash, undefined); + t.is(hash.digest('hex'), crypto.createHash('md5').update('').digest('hex')); +}); + +test('updateHashFromBody() Uint8Array branch', async (t) => { + const hash = crypto.createHash('md5'); + await updateHashFromBody(hash, new Uint8Array(Buffer.from('test'))); + t.is(hash.digest('hex'), crypto.createHash('md5').update('test').digest('hex')); +}); From 3afff4f1c05047674e4cedb995b8b5bfa81ca683 Mon Sep 17 00:00:00 2001 From: Rohan Weeden Date: Tue, 24 Feb 2026 12:25:02 -0500 Subject: [PATCH 16/26] Update CHANGELOG.md to add migration notes for CnmResponse task (#4267) --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a656321c351..15b52bd6fda9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Migration Notes +- **CUMULUS-4395 Core CnmResponse task lambda log group import** + - The lambda function name and log group name for this task are + `-CnmResponse` which might conflict with the non-core version of + the task if you set that up in your terraform. In order to successfully deploy + the core task you may need to either: + - Delete the existing lambda and log group or + - Import the existing lambda and/or log group to allow terraform to modify + them. + ``` + terraform import module.cumulus.module.ingest.module.cnm_response_task.aws_cloudwatch_log_group.cnm_response_task /aws/lambda/-CnmResponse + terraform import module.cumulus.module.ingest.module.cnm_response_task.aws_lambda_function.cnm_response_task arn:aws:lambda:us-east-1::function:-CnmResponse + ``` + + **NOTE: For cumulus core developer ci stacks you only need to import the log + group, since the lambda deployed in the example/cumulus-tf directory will be + renamed automatically.** + ### Notable Changes ### Breaking Changes From 0a888f32fbf7ab92c8bbbffca2b2918e6b434cf1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Feb 2026 18:47:01 +0000 Subject: [PATCH 17/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15b52bd6fda9..319cdf46610d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added `private_api_lambda_arn` output to the archive module and `private_api_lambda_arn` variable to the ingest module. - **CSD-91** - Change update-granules-cmr-metadata-file-links task's updateGranuleIdentifiers flag from true to configurable environment variable `update_granule_identifiers`. - + ### Fixed - **CUMULUS-4566** From 9da0afc784d1998ddd1fce5d3fbfee43bf2c0941 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Tue, 24 Feb 2026 12:02:23 -0800 Subject: [PATCH 18/26] add update_granule_identifiers to tfvars example --- example/cumulus-tf/terraform.tfvars.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/example/cumulus-tf/terraform.tfvars.example b/example/cumulus-tf/terraform.tfvars.example index e20d4c427de9..73f1d4345fd5 100644 --- a/example/cumulus-tf/terraform.tfvars.example +++ b/example/cumulus-tf/terraform.tfvars.example @@ -88,6 +88,10 @@ deploy_cumulus_distribution = true # Set to a value between 1-59000 milliseconds. Recommended: 5000 (5 seconds) for high-concurrency environments. # sync_granule_s3_jitter_max_ms = 5000 +# Optional. Uncomment to change default update granule identifiers value. +# Set to false to prevent update-granules-cmr-metadata-file-links task from updating granule identifer. +# update_granule_identifiers = false + launchpad_api = "launchpadApi" launchpad_certificate = "certificate" launchpad_passphrase = "passphrase" From 54d4df7e0e40056a86eec1bc6eb09a80a800c3c2 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 26 Feb 2026 21:47:53 -0800 Subject: [PATCH 19/26] remove global terraform variable, and move update granule identifier to config. update updateGranulesCmrMetadata with event parameter and pass it updateGranuleIdentifiers from event config to updateEachCmrFileMetadata. update changelog and readme with the new config change --- CHANGELOG.md | 2 +- example/cumulus-tf/main.tf | 3 --- example/cumulus-tf/terraform.tfvars.example | 4 ---- example/cumulus-tf/variables.tf | 6 ------ .../update-granules-cmr-metadata-file-links/README.md | 2 ++ tasks/update-granules-cmr-metadata-file-links/index.js | 10 +++++++--- tf-modules/cumulus/ingest.tf | 3 --- tf-modules/cumulus/variables.tf | 6 ------ .../update-granules-cmr-metadata-file-links-task.tf | 1 - tf-modules/ingest/variables.tf | 6 ------ 10 files changed, 10 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26ba5ccd41e3..a4608d2fc65d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Updated packaging script for granule-invalidator to use `uv pip install` instead of `uv sync`. - Added `private_api_lambda_arn` output to the archive module and `private_api_lambda_arn` variable to the ingest module. - **CSD-91** - - Change update-granules-cmr-metadata-file-links task's updateGranuleIdentifiers flag from true to configurable environment variable `update_granule_identifiers`. + - Changed `update-granules-cmr-metadata-file-links` task config to accept a variable `updateGranuleIdentifiers` for whether or not to granule's `GranuleUr` value. - **CSD-85** - Changed `update-granules-cmr-metadata-file-links` task config to accept a variable `excludeDataGranule` for whether or not to add or update a `Granule.DataGranule` to the granule's metadata, for users who do not want one added or updated from what their granule metadata already is (defaults to `false`). See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details. diff --git a/example/cumulus-tf/main.tf b/example/cumulus-tf/main.tf index 8a6f617c1112..c2a1b73182e9 100644 --- a/example/cumulus-tf/main.tf +++ b/example/cumulus-tf/main.tf @@ -136,9 +136,6 @@ module "cumulus" { default_s3_multipart_chunksize_mb = var.default_s3_multipart_chunksize_mb sync_granule_s3_jitter_max_ms = var.sync_granule_s3_jitter_max_ms - # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not - update_granule_identifiers = var.update_granule_identifiers - launchpad_api = var.launchpad_api launchpad_certificate = var.launchpad_certificate launchpad_passphrase = var.launchpad_passphrase diff --git a/example/cumulus-tf/terraform.tfvars.example b/example/cumulus-tf/terraform.tfvars.example index 73f1d4345fd5..e20d4c427de9 100644 --- a/example/cumulus-tf/terraform.tfvars.example +++ b/example/cumulus-tf/terraform.tfvars.example @@ -88,10 +88,6 @@ deploy_cumulus_distribution = true # Set to a value between 1-59000 milliseconds. Recommended: 5000 (5 seconds) for high-concurrency environments. # sync_granule_s3_jitter_max_ms = 5000 -# Optional. Uncomment to change default update granule identifiers value. -# Set to false to prevent update-granules-cmr-metadata-file-links task from updating granule identifer. -# update_granule_identifiers = false - launchpad_api = "launchpadApi" launchpad_certificate = "certificate" launchpad_passphrase = "passphrase" diff --git a/example/cumulus-tf/variables.tf b/example/cumulus-tf/variables.tf index a53a00c0f22c..d806229c1cdd 100644 --- a/example/cumulus-tf/variables.tf +++ b/example/cumulus-tf/variables.tf @@ -211,12 +211,6 @@ variable "tea_distribution_url" { default = null } -variable "update_granule_identifiers" { - description = "Whether or not to automatically update granule identifiers" - type = bool - default = true -} - variable "sync_granule_s3_jitter_max_ms" { description = "Maximum random jitter in milliseconds for SyncGranule S3 operations (0-59000). Set to 0 to disable." type = number diff --git a/tasks/update-granules-cmr-metadata-file-links/README.md b/tasks/update-granules-cmr-metadata-file-links/README.md index 36bd798965ab..b70639ebd2a5 100644 --- a/tasks/update-granules-cmr-metadata-file-links/README.md +++ b/tasks/update-granules-cmr-metadata-file-links/README.md @@ -11,6 +11,8 @@ This Cumulus task component updates CMR metadata files to have correct values fo **Note** As default behavior for this task, for UMMG and ECHO10 granules, a `Granule.DataGranule` section will be added if not already present within the metadata. For UMMG granules (as of `CSD-85`), the required fields ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/unified-metadata-model/browse/granule/v1.6.6/umm-g-json-schema.json#257)) of `ProductionDateTime` and `DayNightFlag` will be populated with default values (the time the task is ran for `ProductionDateTime` and `Unspecified` for `DayNightFlag`) if they are not already included, along with the other updates described above. For ECHO10 granules, the required field for DataGranules, `ProducerGranuleId` ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/echo-schemas/browse/schemas/10.0/Granule.xsd#409)), will be populated, along with the other updates described above. To disable adding/updating the granule's metadata in this task for both ECHO10 and UMMG granules, set `excludeDataGranule` as `true` (boolean, not a string) (added as part of `CSD-85`) in the task config schema. If set to `true` this task will not change anything relating to the `Granule.DataGranule` in the granule's metadata, no adding or updating. +**Note** To disable updating the granule's identifier in this task for both ECHO10 and UMMG, set `updateGranuleIdentifiers` to `false` (boolean, not a string) in the task config schema (added as part of `CSD-91`). By default, this task sets `updateGranuleIdentifiers` to `true` for both ECHO10 and UMMG. + ## Input/Output Schema ### Input diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index bfee62c6e9b9..0ebf3f554b79 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -51,7 +51,8 @@ async function updateEachCmrFileMetadata( bucketTypes, distributionBucketMap, excludeFileRegexPattern, - excludeDataGranule + excludeDataGranule, + updateGranuleIdentifiers ) { return await Promise.all(cmrFiles.map(async (cmrFile) => { const granuleId = cmrFile.granuleId; @@ -75,7 +76,7 @@ async function updateEachCmrFileMetadata( bucketTypes, cmrGranuleUrlType, distributionBucketMap, - updateGranuleIdentifiers: process.env.update_granule_identifiers === 'true', + updateGranuleIdentifiers, excludeDataGranule, }); })); @@ -118,6 +119,8 @@ async function updateGranulesCmrMetadata(event) { const granulesByGranuleId = keyBy(granules, 'granuleId'); const excludeDataGranule = isBoolean(event.config.excludeDataGranule) ? event.config.excludeDataGranule : false; + const updateGranuleIdentifiers = isBoolean(event.config.updateGranuleIdentifiers) ? + event.config.updateGranuleIdentifiers : true; const distributionBucketMap = await fetchDistributionBucketMap(); const updatedCmrFiles = await updateEachCmrFileMetadata( @@ -128,7 +131,8 @@ async function updateGranulesCmrMetadata(event) { bucketTypes, distributionBucketMap, config.excludeFileRegex, - excludeDataGranule + excludeDataGranule, + updateGranuleIdentifiers ); const updatedGranulesByGranuleId = await updateCmrFileInfo(cmrFiles, granulesByGranuleId); diff --git a/tf-modules/cumulus/ingest.tf b/tf-modules/cumulus/ingest.tf index f9ca53b4963d..8f0f3ac79827 100644 --- a/tf-modules/cumulus/ingest.tf +++ b/tf-modules/cumulus/ingest.tf @@ -75,8 +75,5 @@ module "ingest" { # For message consumer lambdas in order to disable rule/message mismatches allow_provider_mismatch_on_rule_filter = var.allow_provider_mismatch_on_rule_filter - # For update-granules-cmr-metadata-file-links-task in order to update granule identifiers or not - update_granule_identifiers = var.update_granule_identifiers - private_api_lambda_arn = module.archive.private_api_lambda_arn } diff --git a/tf-modules/cumulus/variables.tf b/tf-modules/cumulus/variables.tf index fce082677002..ea10c40acc27 100644 --- a/tf-modules/cumulus/variables.tf +++ b/tf-modules/cumulus/variables.tf @@ -223,12 +223,6 @@ variable "default_s3_multipart_chunksize_mb" { default = 256 } -variable "update_granule_identifiers" { - description = "Whether or not to automatically update granule identifiers" - type = bool - default = true -} - variable "sync_granule_s3_jitter_max_ms" { description = "Maximum random jitter in milliseconds to apply before S3 operations in SyncGranule task (0-59000). Set to 0 to disable jitter." type = number diff --git a/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf b/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf index bd8e2713a15c..24f9ec35538f 100644 --- a/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf +++ b/tf-modules/ingest/update-granules-cmr-metadata-file-links-task.tf @@ -17,7 +17,6 @@ resource "aws_lambda_function" "update_granules_cmr_metadata_file_links_task" { stackName = var.prefix CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" system_bucket = var.system_bucket - update_granule_identifiers = var.update_granule_identifiers } } diff --git a/tf-modules/ingest/variables.tf b/tf-modules/ingest/variables.tf index 3f0a35f67418..2ce157afc4fa 100644 --- a/tf-modules/ingest/variables.tf +++ b/tf-modules/ingest/variables.tf @@ -208,12 +208,6 @@ variable "default_log_retention_days" { description = "Optional default value that user chooses for their log retention periods" } -variable "update_granule_identifiers" { - description = "Whether or not to automatically update granule identifiers" - type = bool - default = true -} - variable "sync_granule_s3_jitter_max_ms" { description = "Maximum random jitter in milliseconds to apply before S3 operations in SyncGranule task (0-59000). Set to 0 to disable jitter." type = number From 5ea4c9085f53463e89924b7cc21a4c0d410ec881 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 26 Feb 2026 23:39:27 -0800 Subject: [PATCH 20/26] add updateGranuleIdentifiers to config correctly. update test from using env var to using config var --- .../schemas/config.json | 5 +++++ .../tests/test-index.js | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tasks/update-granules-cmr-metadata-file-links/schemas/config.json b/tasks/update-granules-cmr-metadata-file-links/schemas/config.json index 5340abb836c9..c861e895911e 100644 --- a/tasks/update-granules-cmr-metadata-file-links/schemas/config.json +++ b/tasks/update-granules-cmr-metadata-file-links/schemas/config.json @@ -49,6 +49,11 @@ "type": "boolean", "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. if True, will disable updates", "default": false + }, + "updateGranuleIdentifiers": { + "type": "boolean", + "description": "Flag to set whether to update granule identifiers as part of this task.", + "default": true } } } diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js index b8a71a6f84a4..8c080b781fc0 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js +++ b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js @@ -340,9 +340,7 @@ test('updateCmrFileInfo - throws error when CMR file not found', async (t) => { }); }); -test('Call to updateGranulesCmrMetadata should be using environment variable to set updateGranuleIdentifier flag', async (t) => { - const originalUpdateGranuleIdentifiers = process.env.update_granule_identifiers; - +test('Call to updateGranulesCmrMetadata should be using config variable to set updateGranuleIdentifier flag', async (t) => { const fakeFetchDistributionBucketMap = sinon.fake.resolves({}); const restoreFetch = updateGranulesCmrMetadataFileLinks.__set__('fetchDistributionBucketMap', fakeFetchDistributionBucketMap); @@ -356,18 +354,20 @@ test('Call to updateGranulesCmrMetadata should be using environment variable to const restoreMapFileEtags = updateGranulesCmrMetadataFileLinks.__set__('mapFileEtags', fakeMapFileEtags); const newPayload = buildPayload(t); + newPayload.config.updateGranuleIdentifiers = false; + await validateConfig(t, newPayload.config); await validateInput(t, newPayload.input); + const filesToUpload = cloneDeep(t.context.filesToUpload); await uploadFiles(filesToUpload, t.context.stagingBucket); - process.env.update_granule_identifiers = false; await updateGranulesCmrMetadataFileLinks.updateGranulesCmrMetadata(newPayload); const fakeUpdateCMRMetadataCheckFalseFlag = fakeUpdateCMRMetadata.firstCall.args[0]; t.false(fakeUpdateCMRMetadataCheckFalseFlag.updateGranuleIdentifiers, 'updateGranuleIdentifiers should be false when the environment variable is set to false'); - process.env.update_granule_identifiers = true; + newPayload.config.updateGranuleIdentifiers = true; fakeUpdateCMRMetadata.resetHistory(); await updateGranulesCmrMetadataFileLinks.updateGranulesCmrMetadata(newPayload); const fakeUpdateCMRMetadataCheckTrueFlag = fakeUpdateCMRMetadata.firstCall.args[0]; @@ -378,6 +378,4 @@ test('Call to updateGranulesCmrMetadata should be using environment variable to restoreMapFileEtags(); restoreUpdateCMRMetadata(); sinon.restore(); - - process.env.update_granule_identifiers = originalUpdateGranuleIdentifiers; }); From ae980cbdad51417adc07559ca858e584aed7ab52 Mon Sep 17 00:00:00 2001 From: wisdomaj <84287378+wisdomaj@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:11:52 -0500 Subject: [PATCH 21/26] CUMULUS-4087 Support Automatic Extension of Cumulus Dashboard User Session (#4231) * added refresh endpoint for launchpad saml * added token extension logic for EDL * Add iat claim to JWT tokens for better client-side session management - Enable the Issued At (iat) claim in createJwtToken by removing the noTimestamp: true option. - Refactor unit tests in test-token.js to handle non-deterministic JWT strings by verifying decoded claims instead of performing direct string comparisons. - This change allows client-side applications to accurately track session age and trigger proactive refreshes. * removed disabled comment since this endpoint is used * Modified oauth refresh token function to decode the existing token and so we can reuse it's iat claim if the token is verified. We then check that the oauth user is authorized before creating a new token and returning it to the client. * Modified saml refresh token function to decode the existing token and so we can reuse it's iat claim if the token is verified * Added iat parameter to createJwtToken function to support reusing iat during token refresh. iat will be used when if the parameter is not null or undefined * added tests to verify expected behavior of JWT refresh * fixed linter issues * added MAX_SESSION_DURATION environment variable and logic to enforce how many times a cumulus dashboard user can extend a session. Added tests to verify this behavior * updated changelog * updated unit tests * updated formatting * consolidated shared logic for token refresh functions in token.js and launchpadSaml.js * fixed linting errors * added unit tests for new token helper functions * set default token refresh expiration time extension to one hour * set default token refresh expiration time extension to one hour --------- Co-authored-by: Austin Wisdom --- CHANGELOG.md | 2 + packages/api/app/routes.js | 1 - packages/api/endpoints/launchpadSaml.js | 59 ++++- packages/api/endpoints/token.js | 113 ++++----- packages/api/lib/token.js | 168 +++++++++++++- .../api/tests/endpoints/test-launchpadSaml.js | 119 +++++++++- packages/api/tests/endpoints/test-token.js | 185 +++++++++++++-- packages/api/tests/lib/test-token.js | 218 +++++++++++++++++- 8 files changed, 754 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4608d2fc65d..87a5a8dae292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -155,6 +155,8 @@ Please complete the following steps before upgrading Cumulus. - **CUMULUS-3574** - Granule file writes are now atomic. Previously, some granule files could be written even if others failed; now, if any granule file fails, none are written. +- **CUMULUS-4087** + - Updated /refresh token endpoint and other functions to support automatic extension of cumulus dashboard user sessions by using iat claims and extending token expiration time. `MAX_SESSION_DURATION` environment variable defaults to 12 hours but can be overriden. - **CUMULUS-4272** - The `tf-modules/cumulus-rds-tf` module now allows specifying an existing security group. This enhancement enables DAACs to migrate their existing RDS deployments to Aurora while diff --git a/packages/api/app/routes.js b/packages/api/app/routes.js index 25a8f76de78a..aa5d58306d40 100644 --- a/packages/api/app/routes.js +++ b/packages/api/app/routes.js @@ -97,7 +97,6 @@ if (launchpadProtectedAuth()) { router.get('/saml/login', launchpadSaml.login); router.post('/saml/auth', launchpadSaml.auth); router.get('/token', launchpadSaml.samlToken); - // disabled for now router.post('/refresh', launchpadSaml.refreshEndpoint); } else { log.info('Using token authentication'); diff --git a/packages/api/endpoints/launchpadSaml.js b/packages/api/endpoints/launchpadSaml.js index 245185ba89e2..ff718df32d95 100644 --- a/packages/api/endpoints/launchpadSaml.js +++ b/packages/api/endpoints/launchpadSaml.js @@ -6,9 +6,7 @@ const path = require('path'); const { JSONPath } = require('jsonpath-plus'); const { parseString } = require('xml2js'); const { promisify } = require('util'); -const get = require('lodash/get'); const moment = require('moment'); - const { getS3Object, parseS3Uri, @@ -18,8 +16,10 @@ const { } = require('@cumulus/aws-client/S3'); const log = require('@cumulus/common/log'); +const get = require('lodash/get'); + const { AccessToken } = require('../models'); -const { createJwtToken } = require('../lib/token'); +const { createJwtToken, refreshTokenAndJwt, verifyAndDecodeTokenFromRequest, handleJwtVerificationError } = require('../lib/token'); const parseXmlString = promisify(parseString); @@ -286,12 +286,55 @@ const samlToken = (req, res) => { return res.redirect(redirectUrl.toString()); }; -const notImplemented = (req, res) => - res.boom.notImplemented( - `endpoint: "${req.path}" not implemented. Login with launchpad.` - ); +/** + * Handle refreshing tokens for SAML authentication + * + * @param {Object} request - an API Gateway request + * @param {Object} response - an API Gateway response object + * @param {number} [extensionSeconds] - number of seconds to extend token + * expiration (default: 43200) + * @returns {Object} an API Gateway response + */ +async function refreshAccessToken(request, response, extensionSeconds = 60 * 60) { + try { + const decodedToken = verifyAndDecodeTokenFromRequest(request); + + const accessTokenModel = new AccessToken(); + + try { + const jwtToken = await refreshTokenAndJwt( + decodedToken, + accessTokenModel, + extensionSeconds + ); + return response.send({ token: jwtToken }); + } catch (error) { + if (error.message === 'Invalid access token') { + return response.boom.unauthorized(error.message); + } + throw error; + } + } catch (error) { + if (error.noToken) { + return response.boom.unauthorized(error.message); + } + if (error.sessionExpired) { + return response.boom.unauthorized(error.message); + } + return handleJwtVerificationError(error.jwtError || error, response); + } +} -const refreshEndpoint = notImplemented; +/** + * Refreshes a SAML token + * + * @param {Object} req - express request object + * @param {Object} res - express response object + * @returns {Promise} the promise of express response object + */ +async function refreshEndpoint(req, res) { + return await refreshAccessToken(req, res); +} module.exports = { auth, diff --git a/packages/api/endpoints/token.js b/packages/api/endpoints/token.js index 9f90d842f103..ff768d2444cc 100644 --- a/packages/api/endpoints/token.js +++ b/packages/api/endpoints/token.js @@ -2,25 +2,21 @@ const get = require('lodash/get'); const log = require('@cumulus/common/log'); -const { RecordDoesNotExist } = require('@cumulus/errors'); const { google } = require('googleapis'); -const { - JsonWebTokenError, - TokenExpiredError, -} = require('jsonwebtoken'); const { EarthdataLoginClient } = require('@cumulus/oauth-client'); -const { - TokenUnauthorizedUserError, -} = require('../lib/errors'); const GoogleOAuth2 = require('../lib/GoogleOAuth2'); const { createJwtToken, + refreshTokenAndJwt, + verifyAndDecodeTokenFromRequest, + handleJwtVerificationError, } = require('../lib/token'); const { verifyJwtAuthorization } = require('../lib/request'); const { AccessToken } = require('../models'); +const { isAuthorizedOAuthUser } = require('../app/auth'); const buildPermanentRedirectResponse = (location, response) => response @@ -28,26 +24,6 @@ const buildPermanentRedirectResponse = (location, response) => .set({ Location: location }) .send('Redirecting'); -/** - * Handle API response for JWT verification errors - * - * @param {Error} err - error thrown by JWT verification - * @param {Object} response - an express response object - * @returns {Promise} the promise of express response object - */ -function handleJwtVerificationError(err, response) { - if (err instanceof TokenExpiredError) { - return response.boom.unauthorized('Access token has expired'); - } - if (err instanceof JsonWebTokenError) { - return response.boom.unauthorized('Invalid access token'); - } - if (err instanceof TokenUnauthorizedUserError) { - return response.boom.unauthorized('User not authorized'); - } - throw err; -} - /** * Handles token requests * @@ -108,60 +84,49 @@ async function token(event, oAuth2Provider, response) { * @param {Object} request - an API Gateway request * @param {OAuth2} oAuth2Provider - an OAuth2 instance * @param {Object} response - an API Gateway response object + * @param {number} [extensionSeconds] - number of seconds to extend token + * expiration (default: 43200) * @returns {Object} an API Gateway response */ -async function refreshAccessToken(request, oAuth2Provider, response) { - const requestJwtToken = get(request, 'body.token'); - - if (!requestJwtToken) { - return response.boom.unauthorized('Request requires a token'); - } - - let accessToken; +async function refreshAccessToken( + request, + oAuth2Provider, + response, + extensionSeconds = 60 * 60 +) { try { - accessToken = await verifyJwtAuthorization(requestJwtToken); - } catch (error) { - return handleJwtVerificationError(error, response); - } + const decodedToken = verifyAndDecodeTokenFromRequest(request); + const username = decodedToken.username; - const accessTokenModel = new AccessToken(); + // Check if the user is authorized (OAuth-specific check) + if (!(await isAuthorizedOAuthUser(username))) { + return response.boom.unauthorized('User not authorized'); + } - let accessTokenRecord; - try { - accessTokenRecord = await accessTokenModel.get({ accessToken }); + const accessTokenModel = new AccessToken(); + + try { + const jwtToken = await refreshTokenAndJwt( + decodedToken, + accessTokenModel, + extensionSeconds + ); + return response.send({ token: jwtToken }); + } catch (error) { + if (error.message === 'Invalid access token') { + return response.boom.unauthorized(error.message); + } + throw error; + } } catch (error) { - if (error instanceof RecordDoesNotExist) { - return response.boom.unauthorized('Invalid access token'); + if (error.noToken) { + return response.boom.unauthorized(error.message); } + if (error.sessionExpired) { + return response.boom.unauthorized(error.message); + } + return handleJwtVerificationError(error.jwtError || error, response); } - - let newAccessToken; - let newRefreshToken; - let expirationTime; - let username; - try { - ({ - accessToken: newAccessToken, - refreshToken: newRefreshToken, - username, - expirationTime, - } = await oAuth2Provider.refreshAccessToken(accessTokenRecord.refreshToken)); - } finally { - // Delete old token record to prevent refresh with old tokens - await accessTokenModel.delete({ - accessToken: accessTokenRecord.accessToken, - }); - } - - // Store new token record - await accessTokenModel.create({ - accessToken: newAccessToken, - refreshToken: newRefreshToken, - expirationTime, - }); - - const jwtToken = createJwtToken({ accessToken: newAccessToken, username, expirationTime }); - return response.send({ token: jwtToken }); } /** diff --git a/packages/api/lib/token.js b/packages/api/lib/token.js index f9d2afd66efb..66b066935583 100644 --- a/packages/api/lib/token.js +++ b/packages/api/lib/token.js @@ -1,13 +1,20 @@ const { sign: jwtSign, verify: jwtVerify } = require('jsonwebtoken'); +const { + JsonWebTokenError, + TokenExpiredError, +} = require('jsonwebtoken'); +const get = require('lodash/get'); +const { RecordDoesNotExist } = require('@cumulus/errors'); +const { TokenUnauthorizedUserError } = require('./errors'); -const createJwtToken = ({ accessToken, expirationTime, username }) => +const createJwtToken = ({ accessToken, expirationTime, username, iat }) => jwtSign({ exp: expirationTime, accessToken, username, + ...(iat && { iat }), }, process.env.TOKEN_SECRET, { algorithm: 'HS256', - noTimestamp: true, }); const verifyJwtToken = (jwtToken, params = {}) => { @@ -27,8 +34,165 @@ function isAccessTokenExpired({ expirationTime }) { return ((Date.now() / 1000) > expirationTime); } +/** + * Gets the maximum session duration from environment variable + * Defaults to 12 hours (43200 seconds) if not set + * + * @returns {number} Maximum session duration in seconds + */ +function getMaxSessionDuration() { + const maxDuration = process.env.MAX_SESSION_DURATION; + if (maxDuration) { + const parsed = Number.parseInt(maxDuration, 10); + if (!Number.isNaN(parsed) && parsed > 0) { + return parsed; + } + } + // Default to 12 hours + return 12 * 60 * 60; +} + +/** + * Checks if a session has exceeded the maximum allowed duration + * + * @param {Object} decodedToken - the decoded JWT token + * @param {number} decodedToken.iat - Issued at timestamp in seconds since epoch + * @returns {boolean} true indicates the session has exceeded max duration + */ +function isSessionExpired(decodedToken) { + const maxSessionDuration = getMaxSessionDuration(); + const currentTime = Math.floor(Date.now() / 1000); + const sessionAge = currentTime - decodedToken.iat; + return sessionAge > maxSessionDuration; +} + +/** + * Verifies and validates a JWT token from a request + * + * Extracts the token from request body, verifies its signature, and checks + * session validity. This consolidates common validation logic used by both + * OAuth and SAML refresh endpoints. + * + * @param {Object} request - the request object containing the token in body.token + * @returns {Promise} the decoded token object + * @throws {Error} with property 'noToken' if token is missing + * @throws {Error} with property 'jwtError' if token verification fails + * @throws {Error} with property 'sessionExpired' if session duration exceeded + */ +function verifyAndDecodeTokenFromRequest(request) { + const requestJwtToken = get(request, 'body.token'); + + if (!requestJwtToken) { + const error = new Error('Request requires a token'); + error.noToken = true; + throw error; + } + + let decodedToken; + try { + decodedToken = verifyJwtToken(requestJwtToken); + } catch (error) { + const err = new Error(`JWT verification failed: ${error.message}`); + err.jwtError = error; + throw err; + } + + // Check if the session has exceeded the maximum duration + if (isSessionExpired(decodedToken)) { + const error = new Error('Session has exceeded maximum duration'); + error.sessionExpired = true; + throw error; + } + + return decodedToken; +} + +/** + * Handle API response for JWT verification errors + * + * Shared error handler for JWT verification failures across all authentication methods. + * + * @param {Error} err - error thrown by JWT verification + * @param {Object} response - an express response object + * @returns {Object} the express response object + */ +function handleJwtVerificationError(err, response) { + if (err instanceof TokenExpiredError) { + return response.boom.unauthorized('Access token has expired'); + } + if (err instanceof JsonWebTokenError) { + return response.boom.unauthorized('Invalid access token'); + } + if (err instanceof TokenUnauthorizedUserError) { + return response.boom.unauthorized('User not authorized'); + } + throw err; +} + +/** + * Refreshes an access token record and returns a new JWT token + * + * Shared logic for token refresh across different authentication methods (OAuth, SAML, etc). + * Handles token verification, expiration checks, database updates, and JWT creation. + * + * @param {Object} decodedToken - the decoded JWT token from the request + * @param {string} decodedToken.accessToken - the access token + * @param {string} decodedToken.username - the username + * @param {number} decodedToken.iat - Issued at timestamp + * @param {AccessToken} accessTokenModel - instance of AccessToken model + * @param {number} [extensionSeconds=43200] - seconds to extend token expiration (default: 12 hours) + * @returns {Promise} the new JWT token + */ +async function refreshTokenAndJwt( + decodedToken, + accessTokenModel, + extensionSeconds = 60 * 60 +) { + const { accessToken, username, iat } = decodedToken; + + // Fetch the access token record from database + let accessTokenRecord; + try { + accessTokenRecord = await accessTokenModel.get({ accessToken }); + } catch (error) { + if (error instanceof RecordDoesNotExist) { + throw new TypeError('Invalid access token'); + } + throw error; + } + + // Use existing token value and extend expiration time + const newAccessToken = accessTokenRecord.accessToken; + + // Extend expiration time by the specified amount (default: 1 hour) + // If expirationTime is undefined, use current time as base + const baseTime = accessTokenRecord.expirationTime || Math.floor(Date.now() / 1000); + const expirationTime = baseTime + extensionSeconds; + + // Update the existing record with new expiration time + await accessTokenModel.update( + { accessToken: accessTokenRecord.accessToken }, + { expirationTime } + ); + + // Preserve the original iat from the token to prevent indefinite authentication + const jwtToken = createJwtToken({ + accessToken: newAccessToken, + username, + expirationTime, + iat, + }); + + return jwtToken; +} + module.exports = { createJwtToken, verifyJwtToken, isAccessTokenExpired, + getMaxSessionDuration, + isSessionExpired, + refreshTokenAndJwt, + verifyAndDecodeTokenFromRequest, + handleJwtVerificationError, }; diff --git a/packages/api/tests/endpoints/test-launchpadSaml.js b/packages/api/tests/endpoints/test-launchpadSaml.js index 1b929d37b848..d0f582ecefef 100644 --- a/packages/api/tests/endpoints/test-launchpadSaml.js +++ b/packages/api/tests/endpoints/test-launchpadSaml.js @@ -17,7 +17,7 @@ const { } = require('@cumulus/aws-client/S3'); const { randomId } = require('@cumulus/common/test-utils'); -const { verifyJwtToken } = require('../../lib/token'); +const { verifyJwtToken, createJwtToken } = require('../../lib/token'); const { AccessToken } = require('../../models'); const launchpadSaml = rewire('../../endpoints/launchpadSaml'); const launchpadPublicCertificate = launchpadSaml.__get__( @@ -363,3 +363,120 @@ test.serial('/token endpoint without API_BASE_URL environment variable returns 5 t.is(response.statusCode, 500); }); + +test.serial('/refresh without a token returns 401 error', async (t) => { + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .expect(401); + + t.is(response.body.message, 'Request requires a token'); +}); + +test.serial('/refresh with an invalid token returns 401 error', async (t) => { + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: 'InvalidToken' }) + .expect(401); + + t.is(response.body.message, 'Invalid access token'); +}); + +test.serial('/refresh with a non-existent token returns 401 error', async (t) => { + const jwt = await buildLaunchpadJwt(t.context.successfulSamlResponse); + // Delete the token from the database to simulate a non-existent token + await accessTokenModel.delete({ accessToken: t.context.validIndex }); + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: jwt }) + .expect(401); + + t.is(response.body.message, 'Invalid access token'); +}); + +test.serial('/refresh with a valid token returns a refreshed token', async (t) => { + const jwt = await buildLaunchpadJwt(t.context.successfulSamlResponse); + const decodedOriginalToken = verifyJwtToken(jwt); + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: jwt }) + .expect(200); + + const decodedRefreshedToken = verifyJwtToken(response.body.token); + t.is(decodedRefreshedToken.username, t.context.validUser); + t.is(decodedRefreshedToken.accessToken, t.context.validIndex); + t.true(decodedRefreshedToken.exp > decodedOriginalToken.exp, 'exp should be extended when refreshing token'); +}); + +test.serial('/refresh preserves the original iat from the token', async (t) => { + const jwt = await buildLaunchpadJwt(t.context.successfulSamlResponse); + const decodedOriginalToken = verifyJwtToken(jwt); + const originalIat = decodedOriginalToken.iat; + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: jwt }) + .expect(200); + + const decodedRefreshedToken = verifyJwtToken(response.body.token); + t.is(decodedRefreshedToken.iat, originalIat, 'iat should be preserved when refreshing token'); + t.not(decodedRefreshedToken.exp, decodedOriginalToken.exp, 'exp should be extended when refreshing token'); +}); + +test.serial('/refresh with an expired session returns 401', async (t) => { + // Set a short max session duration + process.env.MAX_SESSION_DURATION = '3600'; // 1 hour + + const jwt = await buildLaunchpadJwt(t.context.successfulSamlResponse); + const decodedOriginalToken = verifyJwtToken(jwt); + + // Create a new token with an iat that is 2 hours ago (exceeds max session duration) + const oldIat = Math.floor(Date.now() / 1000) - 7200; // 2 hours ago + const expiredJwtToken = createJwtToken({ + accessToken: decodedOriginalToken.accessToken, + username: decodedOriginalToken.username, + expirationTime: decodedOriginalToken.exp, + iat: oldIat, + }); + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: expiredJwtToken }) + .expect(401); + + t.is(response.body.message, 'Session has exceeded maximum duration'); + delete process.env.MAX_SESSION_DURATION; +}); + +test.serial('/refresh with a valid session succeeds', async (t) => { + // Set a short max session duration + process.env.MAX_SESSION_DURATION = '86400'; // 1 day + + const jwt = await buildLaunchpadJwt(t.context.successfulSamlResponse); + const decodedOriginalToken = verifyJwtToken(jwt); + + // Create a new token with an iat that is 1 hour ago (within max session duration) + const recentIat = Math.floor(Date.now() / 1000) - 3600; // 1 hour ago + const recentJwtToken = createJwtToken({ + accessToken: decodedOriginalToken.accessToken, + username: decodedOriginalToken.username, + expirationTime: decodedOriginalToken.exp, + iat: recentIat, + }); + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: recentJwtToken }) + .expect(200); + + t.is(response.status, 200); + delete process.env.MAX_SESSION_DURATION; +}); diff --git a/packages/api/tests/endpoints/test-token.js b/packages/api/tests/endpoints/test-token.js index 1c704cb8e249..004d17a60924 100644 --- a/packages/api/tests/endpoints/test-token.js +++ b/packages/api/tests/endpoints/test-token.js @@ -15,6 +15,8 @@ const { const assertions = require('../../lib/assertions'); const { createJwtToken, + verifyJwtToken, + getMaxSessionDuration, } = require('../../lib/token'); const { fakeAccessTokenFactory, @@ -111,13 +113,13 @@ test.serial('GET /token with an invalid code results in an authorization failure }); test.serial('GET /token with a code but no state returns the access token', async (t) => { + const futureTime = Math.floor(Date.now() / 1000) + 3600; // 1 hour from now const getAccessTokenResponse = { username: 'my-username', accessToken: 'my-access-token', refreshToken: 'my-refresh-token', - expirationTime: 12345, + expirationTime: futureTime, }; - const jwtToken = createJwtToken(getAccessTokenResponse); const stub = sinon.stub( EarthdataLoginClient.prototype, @@ -131,7 +133,10 @@ test.serial('GET /token with a code but no state returns the access token', asyn .expect(200); t.is(response.status, 200); - t.is(response.body.message.token, jwtToken); + const decodedToken = verifyJwtToken(response.body.message.token); + t.is(decodedToken.username, getAccessTokenResponse.username); + t.is(decodedToken.accessToken, getAccessTokenResponse.accessToken); + t.is(decodedToken.exp, getAccessTokenResponse.expirationTime); stub.restore(); }); @@ -159,7 +164,6 @@ test.serial('GET /token with a code and state results in a redirect to that stat test.serial('GET /token with a code and state results in a redirect containing the access token', async (t) => { const getAccessTokenResponse = fakeAccessTokenFactory(); - const jwtToken = createJwtToken(getAccessTokenResponse); const stub = sinon.stub( EarthdataLoginClient.prototype, @@ -178,7 +182,10 @@ test.serial('GET /token with a code and state results in a redirect containing t t.is(locationHeader.origin, 'http://www.example.com'); t.is(locationHeader.pathname, '/state'); - t.is(locationHeader.searchParams.get('token'), jwtToken); + const decodedToken = verifyJwtToken(locationHeader.searchParams.get('token')); + t.is(decodedToken.username, getAccessTokenResponse.username); + t.is(decodedToken.accessToken, getAccessTokenResponse.accessToken); + t.is(decodedToken.exp, getAccessTokenResponse.expirationTime); stub.restore(); }); @@ -253,12 +260,7 @@ test.serial('GET /refresh with an unauthorized user results in an authorization assertions.isUnauthorizedUserResponse(t, response); }); -test.serial('GET /refresh returns 400 if refresh token request fails', async (t) => { - const stub = sinon.stub( - EarthdataLoginClient.prototype, - 'refreshAccessToken' - ).callsFake(() => Promise.reject(new Error('Refresh token request failed'))); - +test.serial('GET /refresh returns 400 if refresh token update fails', async (t) => { const username = randomString(); await setAuthorizedOAuthUsers([username]); @@ -267,6 +269,10 @@ test.serial('GET /refresh returns 400 if refresh token request fails', async (t) const requestJwtToken = createJwtToken(initialTokenRecord); + // Stub the update method to simulate a database failure + const stub = sinon.stub(AccessToken.prototype, 'update') + .callsFake(() => Promise.reject(new Error('Database update failed'))); + const response = await request(app) .post('/refresh') .set('Accept', 'application/json') @@ -277,7 +283,7 @@ test.serial('GET /refresh returns 400 if refresh token request fails', async (t) stub.restore(); }); -test.serial('GET /refresh with a valid token returns a refreshed token', async (t) => { +test.serial('GET /refresh extends token expiration time', async (t) => { const username = randomString(); await setAuthorizedOAuthUsers([username]); @@ -285,14 +291,32 @@ test.serial('GET /refresh with a valid token returns a refreshed token', async ( await accessTokenModel.create(initialTokenRecord); const requestJwtToken = createJwtToken(initialTokenRecord); + const decodedOriginalToken = verifyJwtToken(requestJwtToken); - const refreshedTokenRecord = fakeAccessTokenFactory(); - const refreshedJwtToken = createJwtToken(refreshedTokenRecord); + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: requestJwtToken }) + .expect(200); - const stub = sinon.stub( - EarthdataLoginClient.prototype, - 'refreshAccessToken' - ).callsFake(() => Promise.resolve(refreshedTokenRecord)); + t.is(response.status, 200); + + const decodedRefreshedToken = verifyJwtToken(response.body.token); + // exp should be extended (larger than original) + t.true(decodedRefreshedToken.exp > decodedOriginalToken.exp, 'Expiration time should be extended'); + // username should remain the same + t.is(decodedRefreshedToken.username, username); +}); + +test.serial('GET /refresh with a valid token returns a refreshed token', async (t) => { + const username = randomString(); + await setAuthorizedOAuthUsers([username]); + + const initialTokenRecord = fakeAccessTokenFactory({ username }); + await accessTokenModel.create(initialTokenRecord); + + const requestJwtToken = createJwtToken(initialTokenRecord); + const decodedOriginalToken = verifyJwtToken(requestJwtToken); const response = await request(app) .post('/refresh') @@ -302,15 +326,61 @@ test.serial('GET /refresh with a valid token returns a refreshed token', async ( t.is(response.status, 200); - t.is(response.body.token, refreshedJwtToken); + const decodedToken = verifyJwtToken(response.body.token); + // Username and access token should remain the same since we're just extending expiration + t.is(decodedToken.username, initialTokenRecord.username); + t.is(decodedToken.accessToken, initialTokenRecord.accessToken); + // Expiration should be extended + t.true(decodedToken.exp > decodedOriginalToken.exp); - t.false(await accessTokenModel.exists({ - accessToken: initialTokenRecord.accessToken, - })); + // The original token record should still exist with updated expiration t.true(await accessTokenModel.exists({ - accessToken: refreshedTokenRecord.accessToken, + accessToken: initialTokenRecord.accessToken, })); - stub.restore(); +}); + +test.serial('GET /refresh preserves the original iat from the token', async (t) => { + const username = randomString(); + await setAuthorizedOAuthUsers([username]); + + const initialTokenRecord = fakeAccessTokenFactory({ username }); + await accessTokenModel.create(initialTokenRecord); + + const requestJwtToken = createJwtToken(initialTokenRecord); + const decodedOriginalToken = verifyJwtToken(requestJwtToken); + const originalIat = decodedOriginalToken.iat; + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: requestJwtToken }) + .expect(200); + + t.is(response.status, 200); + + const decodedRefreshedToken = verifyJwtToken(response.body.token); + t.is(decodedRefreshedToken.iat, originalIat, 'iat should be preserved when refreshing token'); + t.not(decodedRefreshedToken.exp, decodedOriginalToken.exp, 'exp should be extended when refreshing token'); +}); + +test.serial('createJwtToken accepts and uses the iat parameter', (t) => { + const username = randomString(); + const accessToken = randomString(); + const expirationTime = Math.floor(Date.now() / 1000) + 3600; + const customIat = Math.floor(Date.now() / 1000) - 1000; // 1000 seconds ago + + const jwtToken = createJwtToken({ + accessToken, + expirationTime, + username, + iat: customIat, + }); + + const decodedToken = verifyJwtToken(jwtToken); + t.is(decodedToken.iat, customIat, 'iat should be set to the custom value'); + t.is(decodedToken.accessToken, accessToken); + t.is(decodedToken.username, username); + t.is(decodedToken.exp, expirationTime); }); test.serial('DELETE /tokenDelete without a token returns a 404 response', async (t) => { @@ -360,3 +430,70 @@ test.serial('DELETE /tokenDelete with a valid token results in a successful dele t.is(response.status, 200); t.is(response.body.message, 'Token record was deleted'); }); + +test.serial('getMaxSessionDuration returns default value when MAX_SESSION_DURATION is not set', (t) => { + delete process.env.MAX_SESSION_DURATION; + const maxDuration = getMaxSessionDuration(); + t.is(maxDuration, 12 * 60 * 60, 'Default should be 12 hours'); +}); + +test.serial('getMaxSessionDuration returns custom value when MAX_SESSION_DURATION is set', (t) => { + process.env.MAX_SESSION_DURATION = '86400'; // 1 day + const maxDuration = getMaxSessionDuration(); + t.is(maxDuration, 86400, 'Should return custom value'); + delete process.env.MAX_SESSION_DURATION; +}); + +test.serial('GET /refresh with an expired session returns 401', async (t) => { + const username = randomString(); + await setAuthorizedOAuthUsers([username]); + + // Set a short max session duration + process.env.MAX_SESSION_DURATION = '3600'; // 1 hour + + const initialTokenRecord = fakeAccessTokenFactory({ username }); + await accessTokenModel.create(initialTokenRecord); + + // Create a token with an iat that is 2 hours ago (exceeds max session duration) + const oldIat = Math.floor(Date.now() / 1000) - 7200; // 2 hours ago + const jwtToken = createJwtToken({ + ...initialTokenRecord, + iat: oldIat, + }); + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: jwtToken }) + .expect(401); + + t.is(response.body.message, 'Session has exceeded maximum duration'); + delete process.env.MAX_SESSION_DURATION; +}); + +test.serial('GET /refresh with a valid session succeeds', async (t) => { + const username = randomString(); + await setAuthorizedOAuthUsers([username]); + + // Set a short max session duration + process.env.MAX_SESSION_DURATION = '43200'; // 12 hours + + const initialTokenRecord = fakeAccessTokenFactory({ username }); + await accessTokenModel.create(initialTokenRecord); + + // Create a token with an iat that is 1 hour ago (within max session duration) + const recentIat = Math.floor(Date.now() / 1000) - 3600; // 1 hour ago + const jwtToken = createJwtToken({ + ...initialTokenRecord, + iat: recentIat, + }); + + const response = await request(app) + .post('/refresh') + .set('Accept', 'application/json') + .send({ token: jwtToken }) + .expect(200); + + t.is(response.status, 200); + delete process.env.MAX_SESSION_DURATION; +}); diff --git a/packages/api/tests/lib/test-token.js b/packages/api/tests/lib/test-token.js index ceffe857895c..0a209a2d87d0 100644 --- a/packages/api/tests/lib/test-token.js +++ b/packages/api/tests/lib/test-token.js @@ -1,8 +1,32 @@ const test = require('ava'); const moment = require('moment'); +const sinon = require('sinon'); +const { + JsonWebTokenError, + TokenExpiredError, +} = require('jsonwebtoken'); +const { RecordDoesNotExist } = require('@cumulus/errors'); +const { TokenUnauthorizedUserError } = require('../../lib/errors'); const { fakeAccessTokenFactory } = require('../../lib/testUtils'); -const { isAccessTokenExpired } = require('../../lib/token'); +const { + createJwtToken, + verifyJwtToken, + isAccessTokenExpired, + getMaxSessionDuration, + isSessionExpired, + refreshTokenAndJwt, + verifyAndDecodeTokenFromRequest, + handleJwtVerificationError, +} = require('../../lib/token'); + +test.beforeEach(() => { + process.env.TOKEN_SECRET = 'secret'; +}); + +test.afterEach(() => { + delete process.env.TOKEN_SECRET; +}); test('isAccessTokenExpired returns true for expired token', (t) => { const accessTokenRecord = fakeAccessTokenFactory({ @@ -24,3 +48,195 @@ test('isAccessTokenExpired returns false for expirationTime with millisecond pre }); t.false(isAccessTokenExpired(accessTokenRecord)); }); + +test('createJwtToken and verifyJwtToken work together', (t) => { + const payload = { accessToken: 'abc', username: 'user', expirationTime: moment().unix() + 100 }; + const token = createJwtToken(payload); + const decoded = verifyJwtToken(token); + t.is(decoded.accessToken, 'abc'); + t.is(decoded.username, 'user'); +}); + +test('createJwtToken accepts and uses the iat parameter', (t) => { + const customIat = Math.floor(Date.now() / 1000) - 1000; + const token = createJwtToken({ + accessToken: 'abc', + expirationTime: moment().unix() + 100, + username: 'user', + iat: customIat, + }); + const decoded = verifyJwtToken(token); + t.is(decoded.iat, customIat); +}); + +test('getMaxSessionDuration returns default value', (t) => { + const oldVal = process.env.MAX_SESSION_DURATION; + delete process.env.MAX_SESSION_DURATION; + t.is(getMaxSessionDuration(), 43200); + process.env.MAX_SESSION_DURATION = oldVal; +}); + +test('getMaxSessionDuration returns custom value', (t) => { + const oldVal = process.env.MAX_SESSION_DURATION; + process.env.MAX_SESSION_DURATION = '3600'; + t.is(getMaxSessionDuration(), 3600); + process.env.MAX_SESSION_DURATION = oldVal; +}); + +test('isSessionExpired returns true if session older than max duration', (t) => { + const oldVal = process.env.MAX_SESSION_DURATION; + process.env.MAX_SESSION_DURATION = '3600'; + const decodedToken = { iat: Math.floor(Date.now() / 1000) - 3601 }; + t.true(isSessionExpired(decodedToken)); + process.env.MAX_SESSION_DURATION = oldVal; +}); + +test('isSessionExpired returns false if session within max duration', (t) => { + const oldVal = process.env.MAX_SESSION_DURATION; + process.env.MAX_SESSION_DURATION = '3600'; + const decodedToken = { iat: Math.floor(Date.now() / 1000) - 3599 }; + t.false(isSessionExpired(decodedToken)); + process.env.MAX_SESSION_DURATION = oldVal; +}); + +test('handleJwtVerificationError handles TokenExpiredError', (t) => { + const response = { + boom: { + unauthorized: sinon.stub().returns('unauthorized'), + }, + }; + const err = new TokenExpiredError('expired', new Date()); + const result = handleJwtVerificationError(err, response); + t.is(result, 'unauthorized'); + t.true(response.boom.unauthorized.calledWith('Access token has expired')); +}); + +test('handleJwtVerificationError handles JsonWebTokenError', (t) => { + const response = { + boom: { + unauthorized: sinon.stub().returns('unauthorized'), + }, + }; + const err = new JsonWebTokenError('invalid'); + const result = handleJwtVerificationError(err, response); + t.is(result, 'unauthorized'); + t.true(response.boom.unauthorized.calledWith('Invalid access token')); +}); + +test('handleJwtVerificationError handles TokenUnauthorizedUserError', (t) => { + const response = { + boom: { + unauthorized: sinon.stub().returns('unauthorized'), + }, + }; + const err = new TokenUnauthorizedUserError('unauthorized'); + const result = handleJwtVerificationError(err, response); + t.is(result, 'unauthorized'); + t.true(response.boom.unauthorized.calledWith('User not authorized')); +}); + +test('handleJwtVerificationError throws if error type unknown', (t) => { + const response = {}; + const err = new Error('unknown'); + t.throws(() => handleJwtVerificationError(err, response)); +}); + +test('verifyAndDecodeTokenFromRequest throws error if token is missing', (t) => { + const request = { body: {} }; + const error = t.throws(() => verifyAndDecodeTokenFromRequest(request)); + t.true(error.noToken); + t.is(error.message, 'Request requires a token'); +}); + +test('verifyAndDecodeTokenFromRequest throws error if token is invalid', (t) => { + const request = { body: { token: 'invalid-token' } }; + const error = t.throws(() => verifyAndDecodeTokenFromRequest(request)); + t.truthy(error.jwtError); + t.true(error.message.includes('JWT verification failed')); +}); + +test('verifyAndDecodeTokenFromRequest throws error if session is expired', (t) => { + const oldMaxDuration = process.env.MAX_SESSION_DURATION; + process.env.MAX_SESSION_DURATION = '3600'; + const iat = Math.floor(Date.now() / 1000) - 4000; + const token = createJwtToken({ accessToken: 'abc', username: 'user', expirationTime: moment().unix() + 100, iat }); + const request = { body: { token } }; + + const error = t.throws(() => verifyAndDecodeTokenFromRequest(request)); + t.true(error.sessionExpired); + t.is(error.message, 'Session has exceeded maximum duration'); + process.env.MAX_SESSION_DURATION = oldMaxDuration; +}); + +test('verifyAndDecodeTokenFromRequest returns decoded token if valid', (t) => { + const iat = Math.floor(Date.now() / 1000) - 100; + const token = createJwtToken({ accessToken: 'abc', username: 'user', expirationTime: moment().unix() + 100, iat }); + const request = { body: { token } }; + + const decoded = verifyAndDecodeTokenFromRequest(request); + t.is(decoded.accessToken, 'abc'); + t.is(decoded.username, 'user'); + t.is(decoded.iat, iat); +}); + +test('refreshTokenAndJwt throws TypeError if token record does not exist', async (t) => { + const accessTokenModel = { + get: sinon.stub().rejects(new RecordDoesNotExist()), + }; + const decodedToken = { accessToken: 'missing', username: 'user', iat: 123 }; + await t.throwsAsync(() => refreshTokenAndJwt(decodedToken, accessTokenModel), { + instanceOf: TypeError, + message: 'Invalid access token', + }); +}); + +test('refreshTokenAndJwt refreshes token and returns new JWT', async (t) => { + const accessToken = 'abc'; + const username = 'user'; + const iat = Math.floor(Date.now() / 1000) - 100; + const initialExpiration = Math.floor(Date.now() / 1000) + 1000; + const extensionSeconds = 3600; + + const accessTokenRecord = { accessToken, expirationTime: initialExpiration }; + const accessTokenModel = { + get: sinon.stub().resolves(accessTokenRecord), + update: sinon.stub().resolves(), + }; + + const decodedToken = { accessToken, username, iat }; + const jwtToken = await refreshTokenAndJwt(decodedToken, accessTokenModel, extensionSeconds); + + t.truthy(jwtToken); + const decoded = verifyJwtToken(jwtToken); + t.is(decoded.accessToken, accessToken); + t.is(decoded.username, username); + t.is(decoded.iat, iat); + t.is(decoded.exp, initialExpiration + extensionSeconds); + + t.true(accessTokenModel.update.calledWith( + { accessToken }, + { expirationTime: initialExpiration + extensionSeconds } + )); +}); + +test('refreshTokenAndJwt uses current time as base if expirationTime is missing', async (t) => { + const accessToken = 'abc'; + const username = 'user'; + const iat = Math.floor(Date.now() / 1000) - 100; + const extensionSeconds = 3600; + + const accessTokenRecord = { accessToken }; // no expirationTime + const accessTokenModel = { + get: sinon.stub().resolves(accessTokenRecord), + update: sinon.stub().resolves(), + }; + + const now = Math.floor(Date.now() / 1000); + const decodedToken = { accessToken, username, iat }; + const jwtToken = await refreshTokenAndJwt(decodedToken, accessTokenModel, extensionSeconds); + + const decoded = verifyJwtToken(jwtToken); + // Should be close to now + extensionSeconds + t.true(decoded.exp >= now + extensionSeconds); + t.true(decoded.exp <= now + extensionSeconds + 2); +}); From 7390d875350011fe0bcc8508de2b8b0e10d105b2 Mon Sep 17 00:00:00 2001 From: Curtis Banh <30607061+cqbanh@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:11:51 -0800 Subject: [PATCH 22/26] cbanh/CUMULUS-4107 (#4254) * Fix/Update ElasticSearch Metrics parameter --- CHANGELOG.md | 2 ++ packages/api/endpoints/logs.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a5a8dae292..da021363a4cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Breaking Changes +- **CUMULUS-4107** + - Changed "_doc" type to "undefined" for ElasticSearch v8.x query parameter . The ES client will omit undefined values from the request. This doesn't touch the other callers. - **CUMULUS-4473** - Updated Granules Bulk Operations API endpoints to accept a list of granuleIds instead of granule objects in the payload. diff --git a/packages/api/endpoints/logs.js b/packages/api/endpoints/logs.js index 30c0056858da..4980b84188ad 100644 --- a/packages/api/endpoints/logs.js +++ b/packages/api/endpoints/logs.js @@ -5,7 +5,7 @@ const { Search } = require('@cumulus/es-client/search'); const metrics = () => ('log_destination_arn' in process.env); -const esMetricsParams = ['_doc', `${process.env.stackName}-*`, true]; +const esMetricsParams = [undefined, `${process.env.stackName}-*`, true]; /** * list all the logs From 81fd19eaa1e97f2d52262492f95c4a1a98384af6 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Mon, 2 Mar 2026 22:36:06 -0800 Subject: [PATCH 23/26] Update echo10 update-granules-cmr-metadata-file-links data file's granuleUr field. add updateGranuleIdentifier test case to cmr-utils. add test case to update-granules-cmr-metadata-file-links that test whether granuleUr is updated or not based on config value --- .../cmrjs/tests/cmr-utils/test-cmr-utils.js | 69 +++++++++++++ .../index.js | 1 + .../tests/data/meta.xml | 2 +- .../tests/test-index.js | 97 ++++++++++++++++++- 4 files changed, 165 insertions(+), 4 deletions(-) diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index 4d741e12fef0..a7571b119a35 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -517,6 +517,40 @@ test.serial('UpdateEcho10XMLMetadata updates GranuleUR and ProducerGranuleID cor t.is(metadataObject.Granule.DataGranule.get('ProducerGranuleId'), 'TestFixtureGranuleUR'); }); +test.serial('UpdateEcho10XMLMetadata should not update GranuleUR and ProducerGranuleID when updateGranuleIdentifiers is false', async (t) => { + const { bucketTypes, distributionBucketMap } = t.context; + const cmrXml = await fs.readFile( + path.join(__dirname, '../fixtures/cmrFileUpdateFixture.cmr.xml'), + 'utf8' + ); + const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions); + // Remove producerGranuleId from cmrMetadata. We expect granuleUR = producerGranuleId + // in this case + // that granuleUR will after = granuleId, and producerGranuleId will be updated + const distEndpoint = 'https://distendpoint.com'; + const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); + + const { metadataObject } = await updateEcho10XMLMetadata({ + cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, + files: [], + distEndpoint, + bucketTypes, + distributionBucketMap, + testOverrides: { + generateEcho10XMLStringMethod: () => 'testXmlString', + metadataObjectFromCMRXMLFileMethod: () => cmrMetadata, + uploadEcho10CMRFileMethod: uploadEchoSpy, + }, + updateGranuleIdentifiers: false, + granuleId: 'TestFixtureGranuleUR_uniq', + producerGranuleId: 'TestFixtureGranuleUR', + excludeDataGranule: false, + }); + + t.deepEqual(metadataObject.Granule.GranuleUR, cmrMetadata.Granule.GranuleUR); + t.deepEqual(metadataObject.Granule.DataGranule, cmrMetadata.Granule.DataGranule); +}); + test.serial('UpdateEcho10XMLMetadata does not update granule DataGranule metadata when excludeDataGranule is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const cmrXml = await fs.readFile( @@ -629,6 +663,41 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); }); +test.serial('updateUMMG Metadata does not updates GranuleUR and ProducerGranuleID when updateGranuleIdentifiers is false', async (t) => { + const { bucketTypes, distributionBucketMap } = t.context; + + const distEndpoint = 'https://distendpoint.com'; + const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); + + const cmrJSON = await fs.readFile( + path.join(__dirname, '../fixtures/MOD09GQ.A3411593.1itJ_e.006.9747594822314_v1.6.2.cmr.json'), + 'utf8' + ); + const cmrMetadata = JSON.parse(cmrJSON); + const filesObject = await readJsonFixture( + path.join(__dirname, '../fixtures/UMMGFilesObjectFixture.json') + ); + + const { metadataObject } = await updateUMMGMetadata({ + granuleId: 'TestFixtureGranuleUR_uniq', + producerGranuleId: 'TestFixtureGranuleUR', + excludeDataGranule: false, + cmrFile: { filename: 's3://cumulus-test-sandbox-private/notUsed' }, + files: filesObject, + distEndpoint, + bucketTypes, + distributionBucketMap, + updateGranuleIdentifiers: false, + testOverrides: { + uploadUMMGJSONCMRFileMethod: uploadEchoSpy, + metadataObjectFromCMRJSONFileMethod: () => cmrMetadata, + }, + }); + + t.deepEqual(metadataObject.GranuleUR, cmrMetadata.GranuleUR); + t.deepEqual(metadataObject.DataGranule, cmrMetadata.DataGranule); +}); + test.serial('updateUMMG Metadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index 0ebf3f554b79..9d712da29770 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -38,6 +38,7 @@ const logger = new Logger({ sender: '@cumulus/update-granules-cmr-metadata-file- * (e.g. { bucket: distribution path }) * @param {Object} excludeFileRegexPattern - pattern by which to exclude files from processing * @param {boolean} excludeDataGranule - Whether to add or update the DataGranule + * @param {boolean} updateGranuleIdentifiers - Whether to update the Granule Identifier * node in the granule's metadata * @returns {Promise} Array of updated CMR files with etags of newly updated files. * diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml b/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml index 8c2a07bf41de..793481c8548c 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml +++ b/tasks/update-granules-cmr-metadata-file-links/tests/data/meta.xml @@ -1,6 +1,6 @@ - MOD11A1.A2017200.h19v04.006.2017201090724 + TestFixtureGranuleUR_uniq 2017-11-20T23:02:40.055807 2017-11-20T23:02:40.055814 diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js index 8c080b781fc0..bca403b9ef6e 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js +++ b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js @@ -4,7 +4,8 @@ const fs = require('fs'); const path = require('path'); const test = require('ava'); const cloneDeep = require('lodash/cloneDeep'); - +const xml2js = require('xml2js'); +const { promisify } = require('util'); const sinon = require('sinon'); const rewire = require('rewire'); const updateGranulesCmrMetadataFileLinks = rewire('../index.js'); @@ -25,7 +26,7 @@ const { const { s3 } = require('@cumulus/aws-client/services'); const { getDistributionBucketMapKey } = require('@cumulus/distribution-utils'); -const { isCMRFile } = require('@cumulus/cmrjs'); +const { isCMRFile, xmlParseOptions } = require('@cumulus/cmrjs'); const { updateGranulesCmrMetadata, updateCmrFileInfo } = require('..'); @@ -340,7 +341,7 @@ test('updateCmrFileInfo - throws error when CMR file not found', async (t) => { }); }); -test('Call to updateGranulesCmrMetadata should be using config variable to set updateGranuleIdentifier flag', async (t) => { +test.serial('Call to updateGranulesCmrMetadata should be using config variable to set updateGranuleIdentifier flag', async (t) => { const fakeFetchDistributionBucketMap = sinon.fake.resolves({}); const restoreFetch = updateGranulesCmrMetadataFileLinks.__set__('fetchDistributionBucketMap', fakeFetchDistributionBucketMap); @@ -379,3 +380,93 @@ test('Call to updateGranulesCmrMetadata should be using config variable to set u restoreUpdateCMRMetadata(); sinon.restore(); }); + +test.serial('GranuleUr does not get updated when config variable to updateGranuleIdentifier flag is false', async (t) => { + const newPayload = buildPayload(t); + + newPayload.input.granules.forEach((granule) => { + const newFile = { + bucket: t.context.publicBucket, + key: 'some/prefix/some_filename.json', + type: 'data', + }; + granule.files.push(newFile); + }); + newPayload.config.updateGranuleIdentifiers = false; + + await validateConfig(t, newPayload.config); + await validateInput(t, newPayload.input); + + const filesToUpload = cloneDeep(t.context.filesToUpload); + + await uploadFiles(filesToUpload, t.context.stagingBucket); + + await updateGranulesCmrMetadata(newPayload); + const cmrFiles = []; + newPayload.input.granules.forEach((granule) => { + granule.files.forEach((file) => { + if (isCMRFile(file)) { + file.granuleId = granule.granuleId; + cmrFiles.push(file); + } + }); + }); + + await Promise.all(cmrFiles.map(async (cmrFile) => { + const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); + const payloadContents = await getObjectStreamContents(payloadResponse.Body); + if (cmrFile.key.endsWith('.xml')) { + const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); + t.not(metaData.Granule.GranuleUR, + cmrFile.granuleId); + } else if (cmrFile.key.endsWith('.json')) { + const metaData = JSON.parse(payloadContents); + t.not(metaData.GranuleUR, cmrFile.granuleId); + } + })); +}); + +test.serial('GranuleUr is updated when config variable to updateGranuleIdentifier flag is true', async (t) => { + const newPayload = buildPayload(t); + + newPayload.input.granules.forEach((granule) => { + const newFile = { + bucket: t.context.publicBucket, + key: 'some/prefix/some_filename.json', + type: 'data', + }; + granule.files.push(newFile); + }); + newPayload.config.updateGranuleIdentifiers = true; + + await validateConfig(t, newPayload.config); + await validateInput(t, newPayload.input); + + const filesToUpload = cloneDeep(t.context.filesToUpload); + + await uploadFiles(filesToUpload, t.context.stagingBucket); + + await updateGranulesCmrMetadata(newPayload); + const cmrFiles = []; + newPayload.input.granules.forEach((granule) => { + granule.files.forEach((file) => { + if (isCMRFile(file)) { + cmrFiles.push(file); + } + }); + }); + + await Promise.all(cmrFiles.map(async (cmrFile) => { + const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); + const payloadContents = await getObjectStreamContents(payloadResponse.Body); + if (cmrFile.key.endsWith('.xml')) { + const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); + t.deepEqual(metaData.Granule.GranuleUR, + metaData.Granule.DataGranule[0].ProducerGranuleId); + } else if (cmrFile.key.endsWith('.json')) { + const metaData = JSON.parse(payloadContents); + t.deepEqual(metaData.GranuleUR, + metaData.DataGranule.Identifiers[0].Identifier); + } + })); +}); From d5b039d4f20152eb000b6787653677c17e7e78e2 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Wed, 4 Mar 2026 12:34:20 -0800 Subject: [PATCH 24/26] Update text based on Naga's feedback Remove sinon test from update-granules-cmr-metadata-file-links test update test based on Naga's feedback --- CHANGELOG.md | 2 +- .../cmrjs/tests/cmr-utils/test-cmr-utils.js | 12 +- .../README.md | 2 +- .../index.js | 4 +- .../tests/test-index.js | 118 +++++++++--------- 5 files changed, 70 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97cde83dd15..bfb1fa3053e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,7 +95,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Updated packaging script for granule-invalidator to use `uv pip install` instead of `uv sync`. - Added `private_api_lambda_arn` output to the archive module and `private_api_lambda_arn` variable to the ingest module. - **CSD-91** - - Changed `update-granules-cmr-metadata-file-links` task config to accept a variable `updateGranuleIdentifiers` for whether or not to granule's `GranuleUr` value. + - Added a task config var to update-granules-cmr-metadata-file-links `updateGranuleIdentifiers` for whether or not to update the Granule metadata's identifiers and `GranuleUR`, defaults to true. See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details. - **CSD-85** - Changed `update-granules-cmr-metadata-file-links` task config to accept a variable `excludeDataGranule` for whether or not to add or update a `Granule.DataGranule` to the granule's metadata, for users who do not want one added or updated from what their granule metadata already is (defaults to `false`). See [update-granules-cmr-metadata-file-links](https://github.com/nasa/cumulus/tree/master/tasks/update-granules-cmr-metadata-file-links#readme) for more details. diff --git a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js index a7571b119a35..4df4467a4406 100644 --- a/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js +++ b/packages/cmrjs/tests/cmr-utils/test-cmr-utils.js @@ -491,8 +491,7 @@ test.serial('UpdateEcho10XMLMetadata updates GranuleUR and ProducerGranuleID cor ); const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions); // Remove producerGranuleId from cmrMetadata. We expect granuleUR = producerGranuleId - // in this case - // that granuleUR will after = granuleId, and producerGranuleId will be updated + // in this case that granuleUR will after = granuleId, and producerGranuleId will be updated const distEndpoint = 'https://distendpoint.com'; const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); @@ -525,8 +524,7 @@ test.serial('UpdateEcho10XMLMetadata should not update GranuleUR and ProducerGra ); const cmrMetadata = await promisify(xml2js.parseString)(cmrXml, xmlParseOptions); // Remove producerGranuleId from cmrMetadata. We expect granuleUR = producerGranuleId - // in this case - // that granuleUR will after = granuleId, and producerGranuleId will be updated + // in this case that granuleUR will after = granuleId, and producerGranuleId will be updated const distEndpoint = 'https://distendpoint.com'; const uploadEchoSpy = sinon.spy(() => Promise.resolve({ ETag: 'foo' })); @@ -629,7 +627,7 @@ test.serial('UpdateEcho10XMLMetadata maintains ECHO10 DataGranule element order' t.is(metadataObject.Granule.DataGranule.get('ProducerGranuleId'), 'NEW_PRODUCER_ID'); }); -test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is true', async (t) => { +test.serial('updateUMMGMetadata updates GranuleUR and ProducerGranuleID correctly when updateGranuleIdentifiers is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const distEndpoint = 'https://distendpoint.com'; @@ -663,7 +661,7 @@ test.serial('updateUMMG Metadata updates GranuleUR and ProducerGranuleID correct t.is(metadataObject.DataGranule.Identifiers[0].Identifier, 'TestFixtureGranuleUR'); }); -test.serial('updateUMMG Metadata does not updates GranuleUR and ProducerGranuleID when updateGranuleIdentifiers is false', async (t) => { +test.serial('updateUMMGMetadata does not update Granule metadata when updateGranuleIdentifiers is false', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const distEndpoint = 'https://distendpoint.com'; @@ -698,7 +696,7 @@ test.serial('updateUMMG Metadata does not updates GranuleUR and ProducerGranuleI t.deepEqual(metadataObject.DataGranule, cmrMetadata.DataGranule); }); -test.serial('updateUMMG Metadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => { +test.serial('updateUMMGMetadata does not update the granule DataGranule metadata when excludeDataGranule is true', async (t) => { const { bucketTypes, distributionBucketMap } = t.context; const distEndpoint = 'https://distendpoint.com'; diff --git a/tasks/update-granules-cmr-metadata-file-links/README.md b/tasks/update-granules-cmr-metadata-file-links/README.md index b70639ebd2a5..91eac7cd57f9 100644 --- a/tasks/update-granules-cmr-metadata-file-links/README.md +++ b/tasks/update-granules-cmr-metadata-file-links/README.md @@ -11,7 +11,7 @@ This Cumulus task component updates CMR metadata files to have correct values fo **Note** As default behavior for this task, for UMMG and ECHO10 granules, a `Granule.DataGranule` section will be added if not already present within the metadata. For UMMG granules (as of `CSD-85`), the required fields ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/unified-metadata-model/browse/granule/v1.6.6/umm-g-json-schema.json#257)) of `ProductionDateTime` and `DayNightFlag` will be populated with default values (the time the task is ran for `ProductionDateTime` and `Unspecified` for `DayNightFlag`) if they are not already included, along with the other updates described above. For ECHO10 granules, the required field for DataGranules, `ProducerGranuleId` ([as seen here](https://git.earthdata.nasa.gov/projects/EMFD/repos/echo-schemas/browse/schemas/10.0/Granule.xsd#409)), will be populated, along with the other updates described above. To disable adding/updating the granule's metadata in this task for both ECHO10 and UMMG granules, set `excludeDataGranule` as `true` (boolean, not a string) (added as part of `CSD-85`) in the task config schema. If set to `true` this task will not change anything relating to the `Granule.DataGranule` in the granule's metadata, no adding or updating. -**Note** To disable updating the granule's identifier in this task for both ECHO10 and UMMG, set `updateGranuleIdentifiers` to `false` (boolean, not a string) in the task config schema (added as part of `CSD-91`). By default, this task sets `updateGranuleIdentifiers` to `true` for both ECHO10 and UMMG. +**Note** To disable updating the granule metadata's identifiers and granuleUr in this task for both ECHO10 and UMMG, set `updateGranuleIdentifiers` to `false` (boolean, not a string) in the task config schema (added as part of `CSD-91`). By default, this task sets `updateGranuleIdentifiers` to `true` for both ECHO10 and UMMG. ## Input/Output Schema diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index 9d712da29770..317f870ad7aa 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -38,8 +38,8 @@ const logger = new Logger({ sender: '@cumulus/update-granules-cmr-metadata-file- * (e.g. { bucket: distribution path }) * @param {Object} excludeFileRegexPattern - pattern by which to exclude files from processing * @param {boolean} excludeDataGranule - Whether to add or update the DataGranule - * @param {boolean} updateGranuleIdentifiers - Whether to update the Granule Identifier - * node in the granule's metadata + * @param {boolean} updateGranuleIdentifiers - Whether to update the Granule's Identifiers + * and granuleUR * @returns {Promise} Array of updated CMR files with etags of newly updated files. * */ diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js index bca403b9ef6e..e6604b67d14d 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js +++ b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js @@ -6,9 +6,6 @@ const test = require('ava'); const cloneDeep = require('lodash/cloneDeep'); const xml2js = require('xml2js'); const { promisify } = require('util'); -const sinon = require('sinon'); -const rewire = require('rewire'); -const updateGranulesCmrMetadataFileLinks = rewire('../index.js'); const { buildS3Uri, @@ -30,6 +27,12 @@ const { isCMRFile, xmlParseOptions } = require('@cumulus/cmrjs'); const { updateGranulesCmrMetadata, updateCmrFileInfo } = require('..'); +function getGranuleURValue(obj) { + const value = obj.GranuleUR ?? obj.Granule?.GranuleUR ?? ''; + // Handle xml2js array wrapping (it converts values to arrays by default) + return Array.isArray(value) ? value[0] : value; +} + function cmrReadStream(file) { return file.endsWith('.cmr.xml') ? fs.createReadStream('tests/data/meta.xml') : fs.createReadStream('tests/data/ummg-meta.json'); } @@ -341,47 +344,7 @@ test('updateCmrFileInfo - throws error when CMR file not found', async (t) => { }); }); -test.serial('Call to updateGranulesCmrMetadata should be using config variable to set updateGranuleIdentifier flag', async (t) => { - const fakeFetchDistributionBucketMap = sinon.fake.resolves({}); - const restoreFetch = updateGranulesCmrMetadataFileLinks.__set__('fetchDistributionBucketMap', fakeFetchDistributionBucketMap); - - const fakeGetObjectSize = sinon.fake.resolves(123); - const restoreGetObjectSize = updateGranulesCmrMetadataFileLinks.__set__('getObjectSize', fakeGetObjectSize); - - const fakeUpdateCMRMetadata = sinon.fake.resolves({}); - const restoreUpdateCMRMetadata = updateGranulesCmrMetadataFileLinks.__set__('updateCMRMetadata', fakeUpdateCMRMetadata); - - const fakeMapFileEtags = sinon.fake.resolves({}); - const restoreMapFileEtags = updateGranulesCmrMetadataFileLinks.__set__('mapFileEtags', fakeMapFileEtags); - - const newPayload = buildPayload(t); - newPayload.config.updateGranuleIdentifiers = false; - - await validateConfig(t, newPayload.config); - await validateInput(t, newPayload.input); - - const filesToUpload = cloneDeep(t.context.filesToUpload); - await uploadFiles(filesToUpload, t.context.stagingBucket); - - await updateGranulesCmrMetadataFileLinks.updateGranulesCmrMetadata(newPayload); - - const fakeUpdateCMRMetadataCheckFalseFlag = fakeUpdateCMRMetadata.firstCall.args[0]; - t.false(fakeUpdateCMRMetadataCheckFalseFlag.updateGranuleIdentifiers, 'updateGranuleIdentifiers should be false when the environment variable is set to false'); - - newPayload.config.updateGranuleIdentifiers = true; - fakeUpdateCMRMetadata.resetHistory(); - await updateGranulesCmrMetadataFileLinks.updateGranulesCmrMetadata(newPayload); - const fakeUpdateCMRMetadataCheckTrueFlag = fakeUpdateCMRMetadata.firstCall.args[0]; - t.true(fakeUpdateCMRMetadataCheckTrueFlag.updateGranuleIdentifiers, 'updateGranuleIdentifiers should be true when the environment variable is set to true'); - - restoreFetch(); - restoreGetObjectSize(); - restoreMapFileEtags(); - restoreUpdateCMRMetadata(); - sinon.restore(); -}); - -test.serial('GranuleUr does not get updated when config variable to updateGranuleIdentifier flag is false', async (t) => { +test.serial('GranuleUr and Identifiers do not get updated when config variable to updateGranuleIdentifier flag is false', async (t) => { const newPayload = buildPayload(t); newPayload.input.granules.forEach((granule) => { @@ -398,8 +361,28 @@ test.serial('GranuleUr does not get updated when config variable to updateGranul await validateInput(t, newPayload.input); const filesToUpload = cloneDeep(t.context.filesToUpload); - await uploadFiles(filesToUpload, t.context.stagingBucket); + const preUpdateCmrFiles = []; + newPayload.input.granules.forEach((granule) => { + granule.files.forEach((file) => { + if (isCMRFile(file)) { + file.granuleId = granule.granuleId; + preUpdateCmrFiles.push(file); + } + }); + }); + + const preUpdateBodyContent = []; + await Promise.all(preUpdateCmrFiles.map(async (cmrFile) => { + const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); + const payloadContents = await getObjectStreamContents(payloadResponse.Body); + if (cmrFile.key.endsWith('.xml')) { + const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); + preUpdateBodyContent.push(metaData); + } else if (cmrFile.key.endsWith('.json')) { + preUpdateBodyContent.push(JSON.parse(payloadContents)); + } + })); await updateGranulesCmrMetadata(newPayload); const cmrFiles = []; @@ -412,23 +395,34 @@ test.serial('GranuleUr does not get updated when config variable to updateGranul }); }); + const updatedBodyContent = []; await Promise.all(cmrFiles.map(async (cmrFile) => { const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); const payloadContents = await getObjectStreamContents(payloadResponse.Body); if (cmrFile.key.endsWith('.xml')) { const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); - t.not(metaData.Granule.GranuleUR, - cmrFile.granuleId); + updatedBodyContent.push(metaData); } else if (cmrFile.key.endsWith('.json')) { - const metaData = JSON.parse(payloadContents); - t.not(metaData.GranuleUR, cmrFile.granuleId); + updatedBodyContent.push(JSON.parse(payloadContents)); } })); + + // sort to avoid test assertion order issues + preUpdateBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b))); + updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b))); + t.is(updatedBodyContent[0].GranuleUR, preUpdateBodyContent[0].GranuleUR); + t.is(updatedBodyContent[1].GranuleUR, preUpdateBodyContent[1].GranuleUR); + t.deepEqual(updatedBodyContent[2].Granule.GranuleUR, + updatedBodyContent[2].Granule.GranuleUR); + t.deepEqual(updatedBodyContent[0].DataGranule, preUpdateBodyContent[0].DataGranule); + t.deepEqual(updatedBodyContent[1].DataGranule, preUpdateBodyContent[1].DataGranule); + t.deepEqual(updatedBodyContent[2].Granule.DataGranule, + updatedBodyContent[2].Granule.DataGranule); }); -test.serial('GranuleUr is updated when config variable to updateGranuleIdentifier flag is true', async (t) => { +test.serial('GranuleUr and Identifiers get updated when config variable to updateGranuleIdentifier flag is true', async (t) => { const newPayload = buildPayload(t); - + newPayload.input.granules.sort((a, b) => a.granuleId.localeCompare(b.granuleId)); newPayload.input.granules.forEach((granule) => { const newFile = { bucket: t.context.publicBucket, @@ -443,10 +437,10 @@ test.serial('GranuleUr is updated when config variable to updateGranuleIdentifie await validateInput(t, newPayload.input); const filesToUpload = cloneDeep(t.context.filesToUpload); - await uploadFiles(filesToUpload, t.context.stagingBucket); await updateGranulesCmrMetadata(newPayload); + const cmrFiles = []; newPayload.input.granules.forEach((granule) => { granule.files.forEach((file) => { @@ -456,17 +450,27 @@ test.serial('GranuleUr is updated when config variable to updateGranuleIdentifie }); }); + const updatedBodyContent = []; await Promise.all(cmrFiles.map(async (cmrFile) => { const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); const payloadContents = await getObjectStreamContents(payloadResponse.Body); if (cmrFile.key.endsWith('.xml')) { const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); - t.deepEqual(metaData.Granule.GranuleUR, - metaData.Granule.DataGranule[0].ProducerGranuleId); + updatedBodyContent.push(metaData); } else if (cmrFile.key.endsWith('.json')) { - const metaData = JSON.parse(payloadContents); - t.deepEqual(metaData.GranuleUR, - metaData.DataGranule.Identifiers[0].Identifier); + updatedBodyContent.push(JSON.parse(payloadContents)); } })); + + updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b))); + t.is(updatedBodyContent[0].GranuleUR, newPayload.input.granules[0].granuleId); + t.is(updatedBodyContent[1].GranuleUR, newPayload.input.granules[1].granuleId); + t.is(updatedBodyContent[2].Granule.GranuleUR[0], + newPayload.input.granules[2].granuleId); + t.is(updatedBodyContent[0].DataGranule.Identifiers[0].Identifier, + newPayload.input.granules[0].granuleId); + t.is(updatedBodyContent[1].DataGranule.Identifiers[0].Identifier, + newPayload.input.granules[1].granuleId); + t.is(updatedBodyContent[2].Granule.DataGranule[0].ProducerGranuleId[0], + newPayload.input.granules[2].granuleId); }); From 7a69485e6bb98e65b3a9ba30ee6c3829ae271f7f Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Wed, 4 Mar 2026 12:35:05 -0800 Subject: [PATCH 25/26] Update variable description based on Naga's review feedback --- .../schemas/config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/update-granules-cmr-metadata-file-links/schemas/config.json b/tasks/update-granules-cmr-metadata-file-links/schemas/config.json index c861e895911e..a5648a834684 100644 --- a/tasks/update-granules-cmr-metadata-file-links/schemas/config.json +++ b/tasks/update-granules-cmr-metadata-file-links/schemas/config.json @@ -47,12 +47,12 @@ }, "excludeDataGranule": { "type": "boolean", - "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. if True, will disable updates", + "description": "Flag to set exclusion behavior of DataGranule metadata updates as part of this task. If true, will disable updates", "default": false }, "updateGranuleIdentifiers": { "type": "boolean", - "description": "Flag to set whether to update granule identifiers as part of this task.", + "description": "Flag to set exclusion behavior of Identifier and granuleUr metadata updates as part of this task. If false, will disable updates", "default": true } } From 38991fcd77e2268a3af40ab9fa2c17b80389f412 Mon Sep 17 00:00:00 2001 From: Allan Yu Date: Thu, 5 Mar 2026 10:50:07 -0800 Subject: [PATCH 26/26] refactor test code for readability fix unintended removal of excludeDataGranule docstring --- .../index.js | 1 + .../tests/test-index.js | 36 +++++++------------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/tasks/update-granules-cmr-metadata-file-links/index.js b/tasks/update-granules-cmr-metadata-file-links/index.js index 317f870ad7aa..6f75f538ad93 100644 --- a/tasks/update-granules-cmr-metadata-file-links/index.js +++ b/tasks/update-granules-cmr-metadata-file-links/index.js @@ -38,6 +38,7 @@ const logger = new Logger({ sender: '@cumulus/update-granules-cmr-metadata-file- * (e.g. { bucket: distribution path }) * @param {Object} excludeFileRegexPattern - pattern by which to exclude files from processing * @param {boolean} excludeDataGranule - Whether to add or update the DataGranule + * node in the granule's metadata * @param {boolean} updateGranuleIdentifiers - Whether to update the Granule's Identifiers * and granuleUR * @returns {Promise} Array of updated CMR files with etags of newly updated files. diff --git a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js index e6604b67d14d..3653fcd6085c 100644 --- a/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js +++ b/tasks/update-granules-cmr-metadata-file-links/tests/test-index.js @@ -33,6 +33,15 @@ function getGranuleURValue(obj) { return Array.isArray(value) ? value[0] : value; } +async function getCMRFileBodyContent(cmrFile) { + const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); + const payloadContents = await getObjectStreamContents(payloadResponse.Body); + if (cmrFile.key.endsWith('.xml')) { + return await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); + } + return JSON.parse(payloadContents); +} + function cmrReadStream(file) { return file.endsWith('.cmr.xml') ? fs.createReadStream('tests/data/meta.xml') : fs.createReadStream('tests/data/ummg-meta.json'); } @@ -374,14 +383,7 @@ test.serial('GranuleUr and Identifiers do not get updated when config variable t const preUpdateBodyContent = []; await Promise.all(preUpdateCmrFiles.map(async (cmrFile) => { - const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); - const payloadContents = await getObjectStreamContents(payloadResponse.Body); - if (cmrFile.key.endsWith('.xml')) { - const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); - preUpdateBodyContent.push(metaData); - } else if (cmrFile.key.endsWith('.json')) { - preUpdateBodyContent.push(JSON.parse(payloadContents)); - } + preUpdateBodyContent.push(await getCMRFileBodyContent(cmrFile)); })); await updateGranulesCmrMetadata(newPayload); @@ -397,14 +399,7 @@ test.serial('GranuleUr and Identifiers do not get updated when config variable t const updatedBodyContent = []; await Promise.all(cmrFiles.map(async (cmrFile) => { - const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); - const payloadContents = await getObjectStreamContents(payloadResponse.Body); - if (cmrFile.key.endsWith('.xml')) { - const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); - updatedBodyContent.push(metaData); - } else if (cmrFile.key.endsWith('.json')) { - updatedBodyContent.push(JSON.parse(payloadContents)); - } + updatedBodyContent.push(await getCMRFileBodyContent(cmrFile)); })); // sort to avoid test assertion order issues @@ -452,14 +447,7 @@ test.serial('GranuleUr and Identifiers get updated when config variable to updat const updatedBodyContent = []; await Promise.all(cmrFiles.map(async (cmrFile) => { - const payloadResponse = await getObject(s3(), { Bucket: cmrFile.bucket, Key: cmrFile.key }); - const payloadContents = await getObjectStreamContents(payloadResponse.Body); - if (cmrFile.key.endsWith('.xml')) { - const metaData = await promisify(xml2js.parseString)(payloadContents, xmlParseOptions); - updatedBodyContent.push(metaData); - } else if (cmrFile.key.endsWith('.json')) { - updatedBodyContent.push(JSON.parse(payloadContents)); - } + updatedBodyContent.push(await getCMRFileBodyContent(cmrFile)); })); updatedBodyContent.sort((a, b) => getGranuleURValue(a).localeCompare(getGranuleURValue(b)));