From c0a07c27bae8bfd45c8bb00f5ab6aae528ad3cac Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 27 Jul 2026 19:52:42 +0200 Subject: [PATCH 1/9] unify stac harvester and stac provider functionality --- .../src/source/gdal_source/reader.rs | 1 - .../gdal_worker_process/process_impl.rs | 23 +- .../gdal_worker_process/process_pool.rs | 18 +- geoengine/sentinel.json | 438 +++++ geoengine/services/src/bin/geoengine-cli.rs | 10 +- geoengine/services/src/cli/mod.rs | 2 + .../src/cli/stac_harvester/discover.rs | 940 +++++++++++ .../src/cli/stac_harvester/harvest.rs | 1491 +++++++++++++++++ .../services/src/cli/stac_harvester/mod.rs | 47 + geoengine/services/src/cli/stac_import.rs | 14 +- .../src/datasets/external/stac/common.rs | 984 +++++++++++ .../datasets/external/stac/loading_info.rs | 92 +- .../src/datasets/external/stac/mod.rs | 1 + .../collections/code-de-minimal.json | 71 + .../collections/landsat-c2-l1-minimal.json | 55 + .../expected-mapping-code-de.json | 69 + .../expected-mapping-landsat-c2-l1.json | 44 + .../items/code-de-harvest-test.json | 200 +++ .../items/landsat-c2-l1-harvest-test.json | 82 + 19 files changed, 4547 insertions(+), 35 deletions(-) create mode 100644 geoengine/sentinel.json create mode 100644 geoengine/services/src/cli/stac_harvester/discover.rs create mode 100644 geoengine/services/src/cli/stac_harvester/harvest.rs create mode 100644 geoengine/services/src/cli/stac_harvester/mod.rs create mode 100644 geoengine/services/src/datasets/external/stac/common.rs create mode 100644 geoengine/test_data/stac_responses/collections/code-de-minimal.json create mode 100644 geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json create mode 100644 geoengine/test_data/stac_responses/expected-mapping-code-de.json create mode 100644 geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json create mode 100644 geoengine/test_data/stac_responses/items/code-de-harvest-test.json create mode 100644 geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json diff --git a/geoengine/operators/src/source/gdal_source/reader.rs b/geoengine/operators/src/source/gdal_source/reader.rs index ebb3710f26..16075b0b26 100644 --- a/geoengine/operators/src/source/gdal_source/reader.rs +++ b/geoengine/operators/src/source/gdal_source/reader.rs @@ -144,7 +144,6 @@ mod tests { CacheHint::default(), ) }) - .map_err(Into::into) } fn tile_information_with_partition_and_shape( diff --git a/geoengine/operators/src/source/gdal_worker_process/process_impl.rs b/geoengine/operators/src/source/gdal_worker_process/process_impl.rs index 8212a44d76..3ca225a3a2 100644 --- a/geoengine/operators/src/source/gdal_worker_process/process_impl.rs +++ b/geoengine/operators/src/source/gdal_worker_process/process_impl.rs @@ -785,7 +785,7 @@ mod tests { let (sender, receiver) = ipc_channel::ipc::channel().unwrap(); - sender.send(msg.clone()).unwrap(); + sender.send(msg).unwrap(); let recv = receiver.recv().unwrap(); assert_eq!(msg, recv); } @@ -797,7 +797,7 @@ mod tests { let (sender, receiver) = ipc_channel::ipc::channel().unwrap(); - sender.send(msg.clone()).unwrap(); + sender.send(msg).unwrap(); let recv = receiver.recv().unwrap(); assert_eq!(msg, recv); } @@ -941,12 +941,12 @@ mod tests { sender.send(msg).unwrap(); let rx_result = receiver .recv() - .inspect_err(|e| panic!("IPC receive: {:?}", e)) + .inspect_err(|e| panic!("IPC receive: {e:?}")) .unwrap(); let payload = match rx_result { Ok(r) => r, - Err(e) => panic!("Error receiving from IPC process: {:?}", e), + Err(e) => panic!("Error receiving from IPC process: {e:?}"), }; let result_2: GdalIpcPayload = (&payload).try_into().unwrap(); @@ -1003,8 +1003,7 @@ mod tests { let dataset = gdc.get_or_open(&dataset_params).unwrap(); let reader_payload = - GdalHandling::load_tile_data::(dataset, &dataset_params, gdal_read_advice) - .map_err(IpcProcessError::from)?; + GdalHandling::load_tile_data::(dataset, &dataset_params, gdal_read_advice)?; let grid_and_props: GridAndProperties = reader_payload.into(); Ok(grid_and_props.into()) @@ -1144,8 +1143,10 @@ mod tests { // file not found => specific error match result { - Err(IpcProcessError::GdalError { kind, .. }) - if kind == IpcProcessGdalErrorKind::FileNotFound => {} + Err(IpcProcessError::GdalError { + kind: IpcProcessGdalErrorKind::FileNotFound, + .. + }) => {} _ => panic!("expected FileNotFound error"), } @@ -1235,8 +1236,10 @@ mod tests { // file not found => specific error match result { - Err(IpcProcessError::GdalError { kind, .. }) - if kind == IpcProcessGdalErrorKind::FileNotFound => {} + Err(IpcProcessError::GdalError { + kind: IpcProcessGdalErrorKind::FileNotFound, + .. + }) => {} _ => panic!("expected FileNotFound error"), } diff --git a/geoengine/operators/src/source/gdal_worker_process/process_pool.rs b/geoengine/operators/src/source/gdal_worker_process/process_pool.rs index 5330d3988a..27d00ded09 100644 --- a/geoengine/operators/src/source/gdal_worker_process/process_pool.rs +++ b/geoengine/operators/src/source/gdal_worker_process/process_pool.rs @@ -1033,7 +1033,7 @@ mod tests { #[test] fn score_exact_match_fresh() { let w = window([100, 100], [200, 200]); - let aff = make_affinity(42, 1, w.clone()); + let aff = make_affinity(42, 1, w); let score = aff.calculate_score(42, 1, &w, aff.timestamp); assert_approx_eq!( @@ -1058,7 +1058,7 @@ mod tests { #[test] fn score_different_band_same_dataset() { let w = window([0, 0], [7, 7]); - let aff = make_affinity(42, 1, w.clone()); + let aff = make_affinity(42, 1, w); let score = aff.calculate_score(42, 2, &w, aff.timestamp); assert_approx_eq!(f64, score, SCORE_DATASET_MATCH); @@ -1067,7 +1067,7 @@ mod tests { #[test] fn score_different_dataset() { let w = window([0, 0], [7, 7]); - let aff = make_affinity(42, 1, w.clone()); + let aff = make_affinity(42, 1, w); let score = aff.calculate_score(99, 1, &w, aff.timestamp); assert_approx_eq!(f64, score, 0.0); @@ -1076,8 +1076,10 @@ mod tests { #[test] fn score_expired_cache() { let w = window([0, 0], [7, 7]); - let mut aff = make_affinity(42, 1, w.clone()); - aff.timestamp = Instant::now() - Duration::from_secs_f64(CACHE_TTL_SECS + 1.0); + let mut aff = make_affinity(42, 1, w); + aff.timestamp = Instant::now() + .checked_sub(Duration::from_secs_f64(CACHE_TTL_SECS + 1.0)) + .unwrap(); let score = aff.calculate_score(42, 1, &w, Instant::now()); assert_approx_eq!(f64, score, 0.0); @@ -1145,8 +1147,8 @@ mod tests { GridIdx2D::new([0, 0]), GridShape2D::new([256, 256]), ), - read_window_bounds: w.clone(), - bounds_of_target: w.clone(), + read_window_bounds: w, + bounds_of_target: w, flip_y: false, }, data_type: RasterDataType::U8, @@ -1287,7 +1289,7 @@ mod tests { let dispatcher = GdalPoolDispatcher::new(pool); let w = window([0, 0], [255, 255]); - let (msg_a, _) = make_request(42, 1, w.clone()); + let (msg_a, _) = make_request(42, 1, w); let (msg_b, _) = make_request(42, 1, w); let d1 = dispatcher.clone(); diff --git a/geoengine/sentinel.json b/geoengine/sentinel.json new file mode 100644 index 0000000000..aab8411031 --- /dev/null +++ b/geoengine/sentinel.json @@ -0,0 +1,438 @@ +{ + "name": "sentinel-2-l2a from STAC", + "id": "11154a6f-ba05-422d-aafc-4812864938dc", + "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.code-de.org/v1", + "priority": 50, + "apiUrl": "https://stac.code-de.org/v1", + "collectionName": "sentinel-2-l2a", + "s3Config": null, + "timeDimension": { + "regular": { + "origin": 0, + "step": { + "granularity": "days", + "step": 1 + } + } + }, + "datasets": [ + { + "name": "sentinel-2-l2a EPSG:32632 U16 20m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { + "x": 20.0, + "y": 20.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 20.0, + "yPixelSize": -20.0 + }, + "gridBounds": { + "min": [ + -34998, + -224999 + ], + "max": [ + 15001, + 275001 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Aerosol optical thickness (AOT) - 20m", + "band_name": null + }, + { + "asset_title": "Blue (band 2) - 20m", + "band_name": null + }, + { + "asset_title": "Coastal aerosol (band 1) - 20m", + "band_name": null + }, + { + "asset_title": "Green (band 3) - 20m", + "band_name": null + }, + { + "asset_title": "NIR 2 (band 8A) - 20m", + "band_name": null + }, + { + "asset_title": "Red (band 4) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 1 (band 5) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 2 (band 6) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 3 (band 7) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 3 (band 7) - 60m", + "band_name": null + }, + { + "asset_title": "SWIR 1 (band 11) - 20m", + "band_name": null + }, + { + "asset_title": "SWIR 2 (band 12) - 20m", + "band_name": null + }, + { + "asset_title": "Water vapour (WVP) - 20m", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U8 10m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U8", + "resolution": { + "x": 10.0, + "y": 10.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 10.0, + "yPixelSize": -10.0 + }, + "gridBounds": { + "min": [ + -69996, + -449998 + ], + "max": [ + 30003, + 550002 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "True color image", + "band_name": "True color image [B04]" + }, + { + "asset_title": "True color image [B02]", + "band_name": null + }, + { + "asset_title": "True color image [B03]", + "band_name": null + }, + { + "asset_title": "True color image [B04]", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U16 10m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { + "x": 10.0, + "y": 10.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 10.0, + "yPixelSize": -10.0 + }, + "gridBounds": { + "min": [ + -69996, + -449998 + ], + "max": [ + 30003, + 550002 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Aerosol optical thickness (AOT) - 10m", + "band_name": null + }, + { + "asset_title": "Blue (band 2) - 10m", + "band_name": null + }, + { + "asset_title": "Green (band 3) - 10m", + "band_name": null + }, + { + "asset_title": "NIR 1 (band 8) - 10m", + "band_name": null + }, + { + "asset_title": "Red (band 4) - 10m", + "band_name": null + }, + { + "asset_title": "Water vapour (WVP) - 10m", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U8 20m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U8", + "resolution": { + "x": 20.0, + "y": 20.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 20.0, + "yPixelSize": -20.0 + }, + "gridBounds": { + "min": [ + -34998, + -224999 + ], + "max": [ + 15001, + 275001 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Cloud probability (CLD) - 20m", + "band_name": "Cloud probability (CLD) - 20m" + }, + { + "asset_title": "Scene classfication map (SCL) - 20m", + "band_name": null + }, + { + "asset_title": "Scene classification map (SCL) - 20m", + "band_name": "Scene classification map (SCL) - 20m" + }, + { + "asset_title": "Snow probability (SNW) - 20m", + "band_name": "Snow probability (SNW) - 20m" + }, + { + "asset_title": "True color image", + "band_name": "True color image [B04]" + }, + { + "asset_title": "True color image [B02]", + "band_name": null + }, + { + "asset_title": "True color image [B03]", + "band_name": null + }, + { + "asset_title": "True color image [B04]", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U16 60m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { + "x": 60.0, + "y": 60.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 60.0, + "yPixelSize": -60.0 + }, + "gridBounds": { + "min": [ + -11666, + -74999 + ], + "max": [ + 5000, + 91667 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Aerosol optical thickness (AOT) - 60m", + "band_name": null + }, + { + "asset_title": "Blue (band 2) - 60m", + "band_name": null + }, + { + "asset_title": "Coastal aerosol (band 1) - 60m", + "band_name": null + }, + { + "asset_title": "Green (band 3) - 60m", + "band_name": null + }, + { + "asset_title": "NIR 2 (band 8A) - 60m", + "band_name": null + }, + { + "asset_title": "NIR 3 (band 9) - 60m", + "band_name": null + }, + { + "asset_title": "Red (band 4) - 60m", + "band_name": null + }, + { + "asset_title": "Red edge 1 (band 5) - 60m", + "band_name": null + }, + { + "asset_title": "Red edge 2 (band 6) - 60m", + "band_name": null + }, + { + "asset_title": "Red edge 3 (band 7) - 60m", + "band_name": "Red edge 3 (band 7) - 60m" + }, + { + "asset_title": "SWIR 1 (band 11) - 60m", + "band_name": null + }, + { + "asset_title": "SWIR 2 (band 12) - 60m", + "band_name": null + }, + { + "asset_title": "Water vapour (WVP) - 60m", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U8 60m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U8", + "resolution": { + "x": 60.0, + "y": 60.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 60.0, + "yPixelSize": -60.0 + }, + "gridBounds": { + "min": [ + -11666, + -74999 + ], + "max": [ + 5000, + 91667 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Cloud probability (CLD) - 60m", + "band_name": "Cloud probability (CLD) - 60m" + }, + { + "asset_title": "Scene classfication map (SCL) - 60m", + "band_name": null + }, + { + "asset_title": "Scene classification map (SCL) - 60m", + "band_name": "Scene classification map (SCL) - 60m" + }, + { + "asset_title": "Snow probability (SNW) - 60m", + "band_name": "Snow probability (SNW) - 60m" + }, + { + "asset_title": "True color image", + "band_name": "True color image [B04]" + }, + { + "asset_title": "True color image [B02]", + "band_name": null + }, + { + "asset_title": "True color image [B03]", + "band_name": null + }, + { + "asset_title": "True color image [B04]", + "band_name": null + } + ] + } + ] +} diff --git a/geoengine/services/src/bin/geoengine-cli.rs b/geoengine/services/src/bin/geoengine-cli.rs index cad4045b21..4e64ffe586 100644 --- a/geoengine/services/src/bin/geoengine-cli.rs +++ b/geoengine/services/src/bin/geoengine-cli.rs @@ -1,8 +1,8 @@ use clap::{Parser, Subcommand}; use geoengine_services::cli::{ - CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, StacImport, - TileImport, check_heartbeat, check_successful_startup, output_openapi_json, - output_toolchain_file, stac_import, tile_import, + CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, StacHarvester, + StacImport, TileImport, check_heartbeat, check_successful_startup, output_openapi_json, + output_toolchain_file, stac_harvester, stac_import, tile_import, }; /// CLI for Geo Engine Utilities @@ -25,6 +25,9 @@ enum Commands { #[command(name = "openapi")] OpenAPI(OpenAPIGenerate), + // Harvests STAC collections using a dataset mapping + StacHarvest(StacHarvester), + // Imports a STAC catalog as a dataset StacImport(StacImport), @@ -42,6 +45,7 @@ impl Commands { Commands::CheckSuccessfulStartup(params) => check_successful_startup(params).await, Commands::Heartbeat(params) => check_heartbeat(params).await, Commands::OpenAPI(params) => output_openapi_json(params).await, + Commands::StacHarvest(params) => stac_harvester(params).await, Commands::StacImport(params) => stac_import(params).await, Commands::TileImport(params) => tile_import(params).await, Commands::ExpressionToolchainFile(params) => output_toolchain_file(params).await, diff --git a/geoengine/services/src/cli/mod.rs b/geoengine/services/src/cli/mod.rs index e8ee25a1cf..80a45f1940 100644 --- a/geoengine/services/src/cli/mod.rs +++ b/geoengine/services/src/cli/mod.rs @@ -2,6 +2,7 @@ mod check_successful_startup; mod expression_toolchain_file; mod heartbeat; mod openapi; +mod stac_harvester; mod stac_import; mod tile_import; @@ -9,5 +10,6 @@ pub use check_successful_startup::{CheckSuccessfulStartup, check_successful_star pub use expression_toolchain_file::{ExpressionToolchainFile, output_toolchain_file}; pub use heartbeat::{Heartbeat, check_heartbeat}; pub use openapi::{OpenAPIGenerate, output_openapi_json}; +pub use stac_harvester::{StacHarvester, stac_harvester}; pub use stac_import::{StacImport, stac_import}; pub use tile_import::{TileImport, tile_import}; diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs new file mode 100644 index 0000000000..b2aec57207 --- /dev/null +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -0,0 +1,940 @@ +use std::{collections::HashMap, path::PathBuf}; + +use anyhow::Context; +use clap::{Parser, ValueEnum}; +use geoengine_datatypes::{ + dataset::DataProviderId, + primitives::SpatialResolution, + raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, + util::Identifier, +}; +use ordered_float::OrderedFloat; +use tracing::{info, warn}; + +use crate::api::model::datatypes::{Measurement, UnitlessMeasurement, UnitlessMeasurementTypeTag}; +use crate::api::model::operators::RasterBandDescriptor; +use crate::datasets::external::stac::{ + StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, StacProviderS3Config, + common, +}; +use geoengine_datatypes::primitives::{ + RegularTimeDimension as DtRegularTimeDimension, TimeDimension as DtTimeDimension, +}; +use geoengine_operators::engine::SpatialGridDescriptor as GeoOpSpatialGridDescriptor; + +// --------------------------------------------------------------------------- +// Discover Mapping +// --------------------------------------------------------------------------- + +/// Probe a STAC collection and sample items to auto-discover the dataset mapping. +#[derive(Debug, Parser)] +pub struct StacDiscoverMapping { + /// STAC API URL + #[arg(long)] + pub stac_url: String, + + /// STAC collection to scan + #[arg(long, default_value = "sentinel-2-l2a")] + pub stac_collection: String, + + /// S3 endpoint (if assets are hosted on S3-compatible storage) + #[arg(long)] + pub s3_endpoint: Option, + + /// S3 access key + #[arg(long)] + pub s3_access_key: Option, + + /// S3 secret key + #[arg(long)] + pub s3_secret_key: Option, + + /// Number of sample items to probe (default: 5) + #[arg(long, default_value_t = 5)] + pub sample_items: usize, + + /// Output file for the mapping JSON (default: stdout) + #[arg(long)] + pub output: Option, + + /// Filter STAC item fields to reduce response size + #[arg(long, default_value_t = false)] + pub filter_item_fields: bool, + + /// File types to import + #[arg(long, value_enum, num_args = 1.., value_delimiter = ' ', default_values_t = [ImportFileType::Cog])] + pub file_types: Vec, + + /// Filter by EPSG codes (only include datasets for these codes) + #[clap(long, value_parser, num_args = 0.., value_delimiter = ' ')] + pub epsgs: Vec, + + /// Verbose output + #[arg(long, default_value_t = false)] + pub verbose: bool, + + /// Bounding box to probe: minx miny maxx maxy (optional, defaults to UTM 32N area) + #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ', default_value = "6.0 47.0 12.0 55.0")] + pub bbox: Option>, + + /// Use the full projected CRS extent for grid bounds instead of the first asset's shape. + /// For UTM projections, this computes grid indices covering the entire zone (easting 0-1,000,000, + /// northing 0-10,000,000) so that multi-tile datasets have correct global raster bounds. + #[arg(long, default_value_t = false)] + pub full_projection_grid: bool, + + /// Time dimension granularity (default: days) + #[arg(long, default_value = "days")] + pub time_granularity: String, + + /// Time dimension step (default: 1) + #[arg(long, default_value_t = 1)] + pub time_step: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum ImportFileType { + Cog, + Jp2, +} + +// --------------------------------------------------------------------------- +// Discover Mapping Implementation +// --------------------------------------------------------------------------- + +#[allow(clippy::too_many_lines)] +pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), anyhow::Error> { + let client = reqwest::Client::new(); + + info!( + "Discovering mapping for STAC collection '{}' at {}", + params.stac_collection, params.stac_url + ); + + let collection_url = format!( + "{}/collections/{}", + params.stac_url.trim_end_matches('/'), + params.stac_collection + ); + + let collection: stac::Collection = stac_api_request_parse(&client, &collection_url) + .await + .context("Failed to fetch STAC collection")?; + + // Scan collection-level item_assets for bands (partial information) + let mut dataset_bands: HashMap> = HashMap::new(); + + for (_asset_key, asset) in &collection.item_assets { + if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + continue; + } + + if let Ok(Some(bands)) = + scan_item_asset_common(&collection.version, asset, collection.summaries.as_ref()) + { + merge_dataset_bands(&mut dataset_bands, bands); + } + } + + if params.verbose { + info!( + "Found {} data type/resolution combinations from collection metadata", + dataset_bands.len() + ); + } + + // Sample items to discover EPSG codes and additional band/resolution info + let items_url = format!( + "{}/collections/{}/items", + params.stac_url.trim_end_matches('/'), + params.stac_collection + ); + + let mut query_params = Vec::new(); + if params.filter_item_fields { + query_params.push(( + "fields".to_string(), + "stac_version,stac_extensions,properties.datetime,properties.updated,properties.proj:epsg,properties.proj:code,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.eo:bands,assets.*.raster:bands,assets.*.proj:epsg,assets.*.proj:code,assets.*.proj:transform,assets.*.proj:shape,assets.*.gsd,assets.*.type".to_string(), + )); + } + if let Some(bbox) = ¶ms.bbox + && bbox.len() == 4 + { + query_params.push(( + "bbox".to_string(), + format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), + )); + } + + query_params.push(("limit".to_string(), format!("{}", params.sample_items))); + + let items_response: stac::ItemCollection = + stac_api_request_with_params(&client, &items_url, &query_params) + .await + .context("Failed to fetch sample items")?; + + if items_response.items.is_empty() { + anyhow::bail!("No items found in the collection. Cannot discover mapping."); + } + + info!( + "Probing {} sample item(s) to discover EPSG codes and additional bands", + items_response.items.len() + ); + + let mut discovered_datasets: HashMap = HashMap::new(); + let mut sample_band_info: HashMap> = HashMap::new(); + + for item in &items_response.items { + let item_epsg = common::epsg_code_from_item(item, common::StacExtensionMajorVersion::V2); + + for (asset_key, asset) in &item.assets { + if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + continue; + } + + let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) + else { + continue; + }; + + let data_type = common::data_type_from_asset_v1_1_0(asset) + .or_else(|| data_type_from_asset_v1_0_0_fallback(asset)); + let Some(data_type) = data_type else { + continue; + }; + + let epsg = common::epsg_code_from_fields( + common::StacExtensionMajorVersion::V2, + &asset.additional_fields, + ) + .or(item_epsg); + let Some(epsg) = epsg else { + continue; + }; + + if !params.epsgs.is_empty() && !params.epsgs.contains(&epsg) { + continue; + } + + let resolution: OrderedFloat = geo_transform.x_pixel_size().abs().into(); + + let dataset_key = DatasetKey { + epsg, + data_type, + resolution, + }; + let partial_key = PartialDatasetKey { + data_type, + resolution, + }; + + let asset_title = asset.title.as_deref().unwrap_or(asset_key).to_string(); + let band_names = common::band_names_from_asset_v1_1_0(asset) + .unwrap_or_else(|_| vec![asset_title.clone()]); + + let entry = sample_band_info.entry(partial_key.clone()).or_default(); + for bn in &band_names { + if !entry.iter().any(|(t, _)| t == &asset_title) { + entry.push((asset_title.clone(), bn.clone())); + } + } + + let info_entry = + discovered_datasets + .entry(dataset_key) + .or_insert(DiscoveredDatasetInfo { + geo_transform: Some(geo_transform), + proj_shape: common::proj_shape_from_fields(&asset.additional_fields), + srs: SpatialReference::new(SpatialReferenceAuthority::Epsg, epsg), + asset_count: 0, + }); + info_entry.asset_count += 1; + } + } + + if discovered_datasets.is_empty() { + anyhow::bail!( + "No matching assets found in sample items. Check your --file-types and --epsgs filters." + ); + } + + // Build the StacDataProviderDefinition + let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) + .map_err(|e| anyhow::anyhow!("{e}"))?; + + let s3_config = params + .s3_endpoint + .as_ref() + .map(|endpoint| StacProviderS3Config { + endpoint: endpoint.clone(), + access_key: params.s3_access_key.clone(), + secret_key: params.s3_secret_key.clone(), + }); + + let mut datasets: Vec = Vec::new(); + + for (dataset_key, info) in &discovered_datasets { + let partial_key = PartialDatasetKey { + data_type: dataset_key.data_type, + resolution: dataset_key.resolution, + }; + + let mut bands: Vec = Vec::new(); + + // Use bands from collection-level scan + if let Some(descriptors) = dataset_bands.get(&partial_key) { + for desc in descriptors { + bands.push(StacProviderDatasetBand { + asset_title: desc.name.clone(), + band_name: None, + }); + } + } + + // Enrich with sample item band info + if let Some(sample_bands) = sample_band_info.get(&partial_key) { + for (asset_title, band_name) in sample_bands { + if !bands.iter().any(|b| b.asset_title == *asset_title) { + bands.push(StacProviderDatasetBand { + asset_title: asset_title.clone(), + band_name: Some(band_name.clone()), + }); + } + } + } + + if bands.is_empty() { + warn!("No bands found for dataset {:?}, skipping", dataset_key); + continue; + } + + bands.sort_by(|a, b| a.asset_title.cmp(&b.asset_title)); + + let spatial_grid = if params.full_projection_grid { + // Compute grid bounds covering the full projected CRS extent + if let Some(gt) = info.geo_transform { + let grid_bounds = projection_grid_bounds(gt, dataset_key.epsg) + .unwrap_or_else(|| { + // Fallback: use first asset's shape + if let Some((height, width)) = info.proj_shape { + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .expect("fallback grid bounds should be valid") + } else { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + } + }); + GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) + } else { + GeoOpSpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0.0, 0.0).into(), + dataset_key.resolution.into_inner(), + -dataset_key.resolution.into_inner(), + ), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid"), + ) + } + } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { + GeoOpSpatialGridDescriptor::source_from_parts( + gt, + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .unwrap_or_else(|_| { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + }), + ) + } else { + GeoOpSpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0.0, 0.0).into(), + dataset_key.resolution.into_inner(), + -dataset_key.resolution.into_inner(), + ), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid"), + ) + }; + + let dataset_name = format!( + "{} EPSG:{} {:?} {}m", + params.stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution + ); + + datasets.push(StacProviderDataset { + name: dataset_name, + description: format!( + "Auto-discovered from STAC collection '{}'", + params.stac_collection + ), + data_type: dataset_key.data_type, + resolution: SpatialResolution::new_unchecked( + dataset_key.resolution.into_inner(), + dataset_key.resolution.into_inner(), + ), + projection: info.srs, + spatial_grid, + bands, + }); + } + + let provider_def = StacDataProviderDefinition { + name: format!("{} from STAC", params.stac_collection), + id: DataProviderId::new(), + description: format!( + "Auto-discovered mapping for STAC collection '{}' at {}", + params.stac_collection, params.stac_url + ), + priority: Some(50), + api_url: params.stac_url.clone(), + collection_name: params.stac_collection.clone(), + s3_config, + time_dimension, + datasets, + }; + + let json = serde_json::to_string_pretty(&provider_def) + .context("Failed to serialize mapping to JSON")?; + + if let Some(output_path) = ¶ms.output { + std::fs::write(output_path, &json) + .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; + println!("Mapping written to {}", output_path.display()); + } else { + println!("{json}"); + } + + Ok(()) +} + +struct DiscoveredDatasetInfo { + geo_transform: Option, + proj_shape: Option<(usize, usize)>, + srs: SpatialReference, + asset_count: u32, +} + +/// Compute grid bounds that cover the full projected CRS extent for the given +/// geo-transform and EPSG code. Currently handles UTM projections (zones 32601–32660 +/// and 32701–32760) with known extents. Returns `None` for unsupported CRS types. +fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option { + let (min_x, max_x, min_y, max_y) = projection_extent(epsg)?; + + let ox = gt.origin_coordinate.x; + let oy = gt.origin_coordinate.y; + let ps_x = gt.x_pixel_size(); + let ps_y = gt.y_pixel_size(); + + // For north-up images ps_y < 0 and origin is top-left. + // Pixel index i = (coord - origin) / pixel_size. + let min_x_idx = ((min_x - ox) / ps_x).floor() as isize; + let max_x_idx = ((max_x - ox) / ps_x).ceil() as isize - 1; + + let (min_y_idx, max_y_idx) = if ps_y < 0.0 { + // ps_y negative: top of area (max_y) → smallest row index + let top = ((max_y - oy) / ps_y).ceil() as isize; + // bottom of area (min_y) → largest row index + let bottom = ((min_y - oy) / ps_y).floor() as isize; + (top, bottom) + } else { + let top = ((max_y - oy) / ps_y).floor() as isize; + let bottom = ((min_y - oy) / ps_y).ceil() as isize - 1; + (top, bottom) + }; + + GridBoundingBox2D::new(GridIdx2D::new([min_x_idx, min_y_idx]), GridIdx2D::new([max_x_idx, max_y_idx])).ok() +} + +/// Return the projected extent `(min_x, max_x, min_y, max_y)` for a given EPSG code. +/// Known extents for UTM zones; other CRS types return `None`. +fn projection_extent(epsg: u32) -> Option<(f64, f64, f64, f64)> { + // UTM northern hemisphere zones EPSG:32601 – 32660 + if (32601..=32660).contains(&epsg) { + return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); + } + // UTM southern hemisphere zones EPSG:32701 – 32760 + if (32701..=32760).contains(&epsg) { + return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); + } + None +} + +/// A key that uniquely identifies a Geo Engine dataset derived from STAC assets. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct DatasetKey { + epsg: u32, + data_type: RasterDataType, + resolution: OrderedFloat, +} + +/// Partial dataset key without EPSG (used during collection scanning). +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct PartialDatasetKey { + data_type: RasterDataType, + resolution: OrderedFloat, +} + +// --------------------------------------------------------------------------- +// Collection scanning helpers +// --------------------------------------------------------------------------- + +fn scan_item_asset_common( + collection_version: &stac::Version, + asset: &stac::ItemAsset, + collection_summaries: Option<&serde_json::Map>, +) -> Result>>, String> { + match collection_version { + stac::Version::v1_0_0 => scan_item_asset_v1_0_0_common(asset), + stac::Version::v1_1_0 => scan_item_asset_v1_1_0_common(asset, collection_summaries), + _ => Err(format!("Unsupported STAC version: {collection_version}")), + } +} + +fn scan_item_asset_v1_0_0_common( + asset: &stac::ItemAsset, +) -> Result>>, String> { + let mut dataset_bands: HashMap> = HashMap::new(); + + let Some(raster_bands) = asset.additional_fields.get("raster:bands") else { + return Ok(None); + }; + let raster_bands: Vec = + serde_json::from_value(raster_bands.clone()) + .map_err(|e| format!("invalid raster:bands: {e}"))?; + + let band_count = raster_bands.len(); + + let eo_bands = asset + .additional_fields + .get("eo:bands") + .and_then(|v| serde_json::from_value::>(v.clone()).ok()); + + if let Some(ref eo_bands_vec) = eo_bands { + if band_count != eo_bands_vec.len() { + return Ok(None); + } + } else if band_count != 1 { + return Ok(None); + } + + for (index, raster_band) in raster_bands.into_iter().enumerate() { + let data_type = raster_band + .data_type + .ok_or_else(|| "Missing data_type in raster band".to_string())?; + let raster_data_type = common::raster_data_type_from_stac_data_type(&data_type) + .ok_or_else(|| format!("Unsupported data type: {data_type:?}"))?; + + let geo_transform = common::geo_transform_from_fields(&asset.additional_fields) + .ok_or_else(|| "Missing proj:transform".to_string())?; + let resolution: OrderedFloat = geo_transform.x_pixel_size().into(); + + let band_name = if let Some(ref eo_bands_vec) = eo_bands { + common::v1_0_0_band_name( + asset.title.as_deref(), + Some(&eo_bands_vec[index]), + band_count, + ) + } else { + common::v1_0_0_band_name(asset.title.as_deref(), None, 1) + }; + + dataset_bands + .entry(PartialDatasetKey { + data_type: raster_data_type, + resolution, + }) + .or_default() + .push(RasterBandDescriptor { + name: band_name, + measurement: Measurement::Unitless(UnitlessMeasurement { + r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, + }), + }); + } + + Ok(Some(dataset_bands)) +} + +fn scan_item_asset_v1_1_0_common( + asset: &stac::ItemAsset, + collection_summaries: Option<&serde_json::Map>, +) -> Result>>, String> { + let mut dataset_bands: HashMap> = HashMap::new(); + + let data_type = asset + .additional_fields + .get("data_type") + .ok_or_else(|| "Missing data_type in asset additional fields".to_string())? + .as_str() + .ok_or_else(|| "data_type is not a string".to_string())?; + + let raster_data_type = common::raster_data_type_from_stac_data_type_str(data_type) + .ok_or_else(|| format!("Unsupported data_type: {data_type}"))?; + + let band_names = common::band_names_from_item_asset_v1_1_0(asset)?; + + let resolution = asset + .additional_fields + .get("gsd") + .and_then(serde_json::Value::as_f64) + .or_else(|| { + common::geo_transform_from_fields(&asset.additional_fields) + .map(|gt| gt.x_pixel_size().abs()) + }) + .or_else(|| { + collection_summaries + .and_then(|s| s.get("gsd")) + .and_then(|v| v.as_array()) + .and_then(|arr| arr.first()) + .and_then(serde_json::Value::as_f64) + }) + .ok_or_else(|| "Missing attribute `gsd` or `proj:transform`".to_string())?; + + for band_name in band_names { + dataset_bands + .entry(PartialDatasetKey { + data_type: raster_data_type, + resolution: resolution.into(), + }) + .or_default() + .push(RasterBandDescriptor { + name: band_name.clone(), + measurement: Measurement::Unitless(UnitlessMeasurement { + r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, + }), + }); + } + + Ok(Some(dataset_bands)) +} + +fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option { + asset + .additional_fields + .get("raster:bands") + .and_then(|v| v.as_array()) + .and_then(|bands| bands.first()) + .and_then(|band| band.get("data_type")) + .and_then(|v| v.as_str()) + .and_then(common::raster_data_type_from_stac_data_type_str) +} + +fn matches_selected_file_types_static( + media_type: Option<&str>, + file_types: &[ImportFileType], +) -> bool { + file_types.iter().any(|file_type| match file_type { + ImportFileType::Cog => common::is_cog_media_type(media_type), + ImportFileType::Jp2 => common::is_jp2_media_type(media_type), + }) +} + +fn merge_dataset_bands( + dataset_bands: &mut HashMap>, + additions: HashMap>, +) { + for (partial_key, band_descriptors) in additions { + let existing_bands = dataset_bands.entry(partial_key).or_default(); + for descriptor in band_descriptors { + if existing_bands.iter().all(|b| b.name != descriptor.name) { + existing_bands.push(descriptor); + } + } + } +} + +fn parse_time_dimension(granularity: &str, step: u64) -> Result { + let dt_granularity = match granularity.to_lowercase().as_str() { + "days" | "day" => geoengine_datatypes::primitives::TimeGranularity::Days, + "months" | "month" => geoengine_datatypes::primitives::TimeGranularity::Months, + "years" | "year" => geoengine_datatypes::primitives::TimeGranularity::Years, + "hours" | "hour" => geoengine_datatypes::primitives::TimeGranularity::Hours, + other => return Err(format!("Unsupported time granularity: {other}")), + }; + + let step_u32: u32 = step + .try_into() + .map_err(|_| format!("step {step} exceeds u32 range"))?; + + Ok(DtTimeDimension::Regular( + DtRegularTimeDimension::new_with_epoch_origin(geoengine_datatypes::primitives::TimeStep { + granularity: dt_granularity, + step: step_u32, + }), + )) +} + +// --------------------------------------------------------------------------- +// STAC API helpers +// --------------------------------------------------------------------------- + +async fn stac_api_request_parse( + client: &reqwest::Client, + url: &str, +) -> Result { + let response = client + .get(url) + .send() + .await + .with_context(|| format!("Failed to fetch {url}"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("STAC API returned HTTP {status}: {body}"); + } + + response + .json() + .await + .with_context(|| format!("Failed to parse response from {url}")) +} + +async fn stac_api_request_with_params( + client: &reqwest::Client, + url: &str, + params: &[(String, String)], +) -> Result { + let response = client + .get(url) + .query(params) + .send() + .await + .with_context(|| format!("Failed to fetch {url}"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("STAC API returned HTTP {status}: {body}"); + } + + response + .json() + .await + .with_context(|| format!("Failed to parse response from {url}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use httptest::{Expectation, Server, all_of, matchers::request, responders}; + + const COLLECTION_PATH: &str = "/v1/collections/sentinel-2-l2a"; + const ITEMS_PATH: &str = "/v1/collections/sentinel-2-l2a/items"; + + fn stac_collection_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/collections/code-de-minimal.json" + )) + .expect("valid collection fixture") + } + + fn stac_items_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/code-de-harvest-test.json" + )) + .expect("valid items fixture") + } + + fn expected_mapping_json() -> serde_json::Value { + let mut mapping: serde_json::Value = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid expected mapping fixture"); + // Remove the id since it is auto-generated + mapping.as_object_mut().unwrap().remove("id"); + mapping + } + + #[tokio::test] + async fn test_discover_mapping_produces_expected_mapping() { + let stac_server = Server::run(); + + // Mock collection endpoint + stac_server.expect( + Expectation::matching(request::method_path("GET", COLLECTION_PATH)) + .times(1) + .respond_with(responders::json_encoded(stac_collection_json())), + ); + + // Mock items endpoint + stac_server.expect( + Expectation::matching(all_of![request::method("GET"), request::path(ITEMS_PATH),]) + .times(1) + .respond_with(responders::json_encoded(stac_items_json())), + ); + + let output_path = std::env::temp_dir().join("test_discover_mapping_output.json"); + + let params = StacDiscoverMapping { + stac_url: stac_server.url_str("/v1").trim_end_matches('/').to_string(), + stac_collection: "sentinel-2-l2a".to_string(), + s3_endpoint: None, + s3_access_key: None, + s3_secret_key: None, + sample_items: 2, + output: Some(output_path.clone()), + filter_item_fields: true, + file_types: vec![ImportFileType::Jp2], + epsgs: vec![], + bbox: None, + full_projection_grid: false, + verbose: false, + time_granularity: "days".to_string(), + time_step: 1, + }; + + discover_mapping(params) + .await + .expect("discover mapping should succeed"); + + assert!(output_path.exists(), "output mapping file should exist"); + + let output_content = + std::fs::read_to_string(&output_path).expect("should read output file"); + let mut output_json: serde_json::Value = + serde_json::from_str(&output_content).expect("output should be valid JSON"); + + // Normalize dynamic fields before comparison + let output_obj = output_json.as_object_mut().unwrap(); + output_obj.remove("id"); + output_obj.insert( + "apiUrl".to_string(), + serde_json::json!("https://stac.test/v1"), + ); + output_obj.insert("description".to_string(), serde_json::json!("Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.test/v1")); + + // Sort datasets by name for deterministic comparison (HashMap order) + if let Some(datasets) = output_obj + .get_mut("datasets") + .and_then(|d| d.as_array_mut()) + { + datasets.sort_by(|a, b| { + a["name"] + .as_str() + .unwrap_or("") + .cmp(b["name"].as_str().unwrap_or("")) + }); + } + + let expected = expected_mapping_json(); + + pretty_assertions::assert_eq!(expected, output_json); + + // Clean up + let _ = std::fs::remove_file(&output_path); + } + + // ----------------------------------------------------------------------- + // Landsat C2 L1 discover test + // ----------------------------------------------------------------------- + + fn landsat_collection_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/collections/landsat-c2-l1-minimal.json" + )) + .expect("valid Landsat collection fixture") + } + + fn landsat_items_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/landsat-c2-l1-harvest-test.json" + )) + .expect("valid Landsat items fixture") + } + + fn expected_landsat_mapping_json() -> serde_json::Value { + let mut mapping: serde_json::Value = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" + )) + .expect("valid Landsat expected mapping fixture"); + mapping.as_object_mut().unwrap().remove("id"); + mapping + } + + const LANDSAT_COLLECTION_PATH: &str = "/v1/collections/landsat-c2-l1"; + const LANDSAT_ITEMS_PATH: &str = "/v1/collections/landsat-c2-l1/items"; + + #[tokio::test] + async fn test_discover_landsat_mapping_produces_expected_mapping() { + let stac_server = Server::run(); + + stac_server.expect( + Expectation::matching(request::method_path("GET", LANDSAT_COLLECTION_PATH)) + .times(1) + .respond_with(responders::json_encoded(landsat_collection_json())), + ); + + stac_server.expect( + Expectation::matching(all_of![request::method("GET"), request::path(LANDSAT_ITEMS_PATH)]) + .times(1) + .respond_with(responders::json_encoded(landsat_items_json())), + ); + + let output_path = std::env::temp_dir().join("test_discover_landsat_mapping_output.json"); + + let params = StacDiscoverMapping { + stac_url: stac_server.url_str("/v1").trim_end_matches('/').to_string(), + stac_collection: "landsat-c2-l1".to_string(), + s3_endpoint: None, + s3_access_key: None, + s3_secret_key: None, + sample_items: 1, + output: Some(output_path.clone()), + filter_item_fields: true, + file_types: vec![ImportFileType::Cog], + epsgs: vec![], + bbox: None, + full_projection_grid: false, + verbose: false, + time_granularity: "days".to_string(), + time_step: 1, + }; + + discover_mapping(params) + .await + .expect("Landsat discover mapping should succeed"); + + assert!(output_path.exists(), "output mapping file should exist"); + + let output_content = + std::fs::read_to_string(&output_path).expect("should read output file"); + let mut output_json: serde_json::Value = + serde_json::from_str(&output_content).expect("output should be valid JSON"); + + let output_obj = output_json.as_object_mut().unwrap(); + output_obj.remove("id"); + output_obj.insert( + "apiUrl".to_string(), + serde_json::json!("https://stac.test/v1"), + ); + output_obj.insert("description".to_string(), serde_json::json!("Auto-discovered mapping for STAC collection 'landsat-c2-l1' at https://stac.test/v1")); + + if let Some(datasets) = output_obj + .get_mut("datasets") + .and_then(|d| d.as_array_mut()) + { + datasets.sort_by(|a, b| { + a["name"] + .as_str() + .unwrap_or("") + .cmp(b["name"].as_str().unwrap_or("")) + }); + } + + let expected = expected_landsat_mapping_json(); + + pretty_assertions::assert_eq!(expected, output_json); + + let _ = std::fs::remove_file(&output_path); + } +} diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs new file mode 100644 index 0000000000..19fb812932 --- /dev/null +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -0,0 +1,1491 @@ +use std::{ + collections::HashMap, + io::Read, + str::FromStr, + time::{Duration, Instant}, +}; + +use anyhow::Context; +use chrono::Timelike; +use futures::StreamExt; +use geoengine_datatypes::{ + dataset::NamedData, + primitives::{DateTime, TimeInstance, TimeInterval}, + raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority, SpatialReferenceOption}, +}; +use tracing::{debug, error, info, warn}; + +use crate::datasets::external::stac::{StacDataProviderDefinition, StacProviderDataset, common}; +use crate::{ + api::{ + handlers::{ + datasets::AddDatasetTile, + permissions::{ + DatasetResource, DatasetResourceTypeTag, LayerCollectionResource, + LayerCollectionResourceTypeTag, LayerResource, LayerResourceTypeTag, + PermissionRequest, Resource, + }, + }, + model::{ + datatypes::{ + GdalConfigOption, GridBoundingBox2D as ApiGridBoundingBox2D, + GridIdx2D as ApiGridIdx2D, LayerId, Measurement, SpatialGridDefinition, + TimeGranularity, TimeStep, UnitlessMeasurement, UnitlessMeasurementTypeTag, + }, + operators::{ + GdalDatasetParameters, GdalMultiBand, GdalMultiBandTypeTag, RasterBandDescriptor, + RasterBandDescriptors, RasterResultDescriptor, RegularTimeDimension, + SpatialGridDescriptor, SpatialGridDescriptorState, TimeDescriptor, TimeDimension, + }, + responses::{ErrorResponse, IdResponse}, + services::{ + AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, + }, + }, + }, + datasets::{DatasetName, upload::VolumeName}, + layers::{ + layer::{AddLayer, AddLayerCollection, CollectionItem, LayerCollection}, + listing::LayerCollectionId, + storage::{INTERNAL_LAYER_DB_ROOT_COLLECTION_ID, INTERNAL_PROVIDER_ID}, + }, + permissions::{Permission, Role}, + workflows::workflow::Workflow, +}; +use geoengine_operators::{ + engine::{RasterOperator, TypedOperator}, + source::{MultiBandGdalSource, MultiBandGdalSourceParameters}, +}; + +// --------------------------------------------------------------------------- +// Harvest +// --------------------------------------------------------------------------- + +/// Harvest tiles from a STAC collection using a predefined dataset mapping. +#[derive(Debug, clap::Parser)] +pub struct StacHarvest { + /// Path to the `StacDataProviderDefinition` JSON file (or `-` for stdin) + #[arg(long, value_parser = parse_mapping_file)] + pub mapping: StacDataProviderDefinition, + + /// Time range start to import (optional) + #[arg(long)] + pub time_start: Option, + + /// Time range end to import (optional) + #[arg(long)] + pub time_end: Option, + + /// Bounding box to import: minx miny maxx maxy (optional) + #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ')] + pub bbox: Option>, + + /// Import limit (page size) + #[arg(long)] + pub limit: Option, + + /// Geo Engine API URL + #[arg(long, default_value = "http://localhost:3030/api")] + pub geo_engine_url: String, + + /// Geo Engine API email + #[arg(long, default_value = "admin@localhost")] + pub geo_engine_email: String, + + /// Geo Engine API password + #[arg(long, default_value = "adminadmin")] + pub geo_engine_password: String, + + /// Volume on the server + #[arg(long, default_value = "geodata")] + pub volume_name: String, + + /// Verbose output + #[arg(long, default_value_t = false)] + pub verbose: bool, + + /// Number of pages to prefetch while processing the current page + #[arg(long, default_value_t = 2)] + pub prefetch_pages: usize, + + /// Z-index property name + #[arg(long, default_value = "updated")] + pub z_index_property_name: Option, + + /// No data value override + #[arg(long)] + pub no_data_value: Option, + + /// GDAL retry count + #[arg(long)] + pub gdal_retries: Option, + + /// Filter item fields to reduce response size + #[arg(long, default_value_t = false)] + pub filter_item_fields: bool, +} + +/// Parse a JSON file path (or `-` for stdin) into a `StacDataProviderDefinition`. +fn parse_mapping_file(s: &str) -> Result { + let json = if s == "-" { + let mut input = String::new(); + std::io::stdin() + .read_to_string(&mut input) + .map_err(|e| format!("Failed to read mapping from stdin: {e}"))?; + input + } else { + std::fs::read_to_string(s).map_err(|e| format!("Failed to read mapping from '{s}': {e}"))? + }; + serde_json::from_str(&json).map_err(|e| format!("Invalid mapping JSON: {e}")) +} + +// --------------------------------------------------------------------------- +// Harvest Implementation +// --------------------------------------------------------------------------- + +#[allow(clippy::too_many_lines)] +pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Error> { + let start_time = Instant::now(); + + let provider_def = ¶ms.mapping; + + info!( + "Harvesting STAC collection '{}' at {} with {} dataset(s)", + provider_def.collection_name, + provider_def.api_url, + provider_def.datasets.len() + ); + + let (client, session_id) = login_geo_engine( + ¶ms.geo_engine_url, + ¶ms.geo_engine_email, + ¶ms.geo_engine_password, + ) + .await?; + + let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); + + for (idx, dataset) in provider_def.datasets.iter().enumerate() { + let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + + if params.verbose { + info!("Checking dataset '{}'", dataset_name); + } + + if !dataset_exists_api(&client, ¶ms.geo_engine_url, &session_id, &dataset_name).await? { + create_dataset_api( + &client, + ¶ms.geo_engine_url, + &session_id, + &dataset_name, + dataset, + ¶ms.volume_name, + ) + .await?; + created_datasets.push((idx, dataset.clone())); + } + } + + info!( + "Created {} new dataset(s) out of {}", + created_datasets.len(), + provider_def.datasets.len() + ); + + let stac_api_url = provider_def.api_url.trim_end_matches('/').to_string(); + let items_url = format!( + "{}/collections/{}/items", + stac_api_url, provider_def.collection_name + ); + + let mut query_params: Vec<(String, String)> = Vec::new(); + + if let Some(bbox) = ¶ms.bbox + && bbox.len() == 4 + { + query_params.push(( + "bbox".to_string(), + format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), + )); + } + + if params.time_start.is_some() || params.time_end.is_some() { + query_params.push(( + "datetime".to_string(), + format!( + "{}/{}", + params.time_start.as_deref().unwrap_or(""), + params.time_end.as_deref().unwrap_or("") + ), + )); + } + + if let Some(limit) = params.limit { + query_params.push(("limit".to_string(), limit.to_string())); + } + + if params.filter_item_fields { + query_params.push(( + "fields".to_string(), + "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" + .to_string(), + )); + } + + let initial_query_state = QueryState::FirstPage { + query_url: items_url.clone(), + query_params, + }; + + let page_stream = create_page_stream( + initial_query_state, + client.clone(), + params.verbose, + params.prefetch_pages, + ); + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut dynamic_datasets: HashMap = HashMap::new(); + let mut items_processed: u64 = 0; + let mut items_per_sec: f64; + + futures::pin_mut!(page_stream); + while let Some(result) = page_stream.next().await { + let item_collection = result?; + + for item in &item_collection.items { + process_harvest_item_dynamic( + item, + provider_def, + &mut tiles_by_dataset, + &mut dynamic_datasets, + ¶ms, + ) + .unwrap_or_else(|e| { + if params.verbose { + warn!("Skipping item {}: {}", item.id, e); + } + }); + + items_processed += 1; + } + + if params.verbose { + let elapsed = start_time.elapsed().as_secs_f64(); + items_per_sec = if elapsed > 0.0 { + items_processed as f64 / elapsed + } else { + 0.0 + }; + + if let Some(number_matched) = item_collection + .additional_fields + .get("numberMatched") + .and_then(serde_json::Value::as_u64) + { + let progress = + (items_processed as f64 / number_matched as f64 * 100.0).clamp(0.0, 100.0); + let remaining = number_matched.saturating_sub(items_processed); + let eta_secs = if items_per_sec > 0.0 { + remaining as f64 / items_per_sec + } else { + f64::INFINITY + }; + let eta_str = if eta_secs.is_finite() { + format_duration(eta_secs as u64) + } else { + "unknown".to_string() + }; + println!( + "[{progress:.1}%] Processed {items_processed}/{number_matched} items ({items_per_sec:.1} items/s, ETA: {eta_str})" + ); + } else { + println!("Processed {items_processed} items ({items_per_sec:.1} items/s)"); + } + } + } + + info!("Processed {} items total", items_processed); + + // Create any dynamic (per-EPSG) datasets that were discovered during item processing + for (dyn_dataset_name, dyn_dataset) in &dynamic_datasets { + if !dataset_exists_api( + &client, + ¶ms.geo_engine_url, + &session_id, + dyn_dataset_name, + ) + .await? + { + if params.verbose { + info!("Creating dynamic dataset '{}'", dyn_dataset_name); + } + create_dataset_api( + &client, + ¶ms.geo_engine_url, + &session_id, + dyn_dataset_name, + dyn_dataset, + ¶ms.volume_name, + ) + .await?; + } + // Also ensure tiles_by_dataset has an entry for this dataset + tiles_by_dataset + .entry(dyn_dataset_name.clone()) + .or_default(); + } + + for (dataset_name, tiles) in &tiles_by_dataset { + if tiles.is_empty() { + continue; + } + + if params.verbose { + info!("Adding {} tiles to dataset '{}'", tiles.len(), dataset_name); + } + + let batch_size = 100; + for chunk in tiles.chunks(batch_size) { + let response = retry_http( + || async { + client + .post(format!( + "{}/dataset/{}/tiles", + params.geo_engine_url, dataset_name + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(chunk) + .send() + .await + }, + &format!("Add tiles to dataset '{dataset_name}'"), + ) + .await + .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + warn!("Failed to add tiles to dataset '{dataset_name}' (HTTP {status}): {body}"); + // Continue with remaining tiles; some conflicts (e.g. z-index) are expected + } + } + } + + create_harvest_layer_collections( + &client, + ¶ms.geo_engine_url, + &session_id, + provider_def, + &created_datasets, + ¶ms, + ) + .await?; + + let elapsed = start_time.elapsed(); + info!("Harvest completed in {:.2?}", elapsed); + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Item Processing (Harvest) +// --------------------------------------------------------------------------- + +/// Like `process_harvest_item`, but handles items whose EPSG code differs from +/// the mapping's projection by dynamically creating per-EPSG dataset variants. +/// Items that pass the `--epsgs` filter but have a different EPSG than the +/// mapping will be grouped into separate datasets named with their actual EPSG. +#[allow(clippy::too_many_lines)] +fn process_harvest_item_dynamic( + item: &stac::Item, + provider_def: &StacDataProviderDefinition, + tiles_by_dataset: &mut HashMap>, + dynamic_datasets: &mut HashMap, + params: &StacHarvest, +) -> Result<(), anyhow::Error> { + let Some(datetime) = item.properties.datetime else { + return Ok(()); + }; + + let date_without_time = datetime + .with_hour(0) + .and_then(|d| d.with_minute(0)) + .and_then(|d| d.with_second(0)) + .and_then(|d| d.with_nanosecond(0)) + .context("Failed to set time to zero")?; + let date_without_time: DateTime = date_without_time.into(); + let time: TimeInstance = date_without_time.into(); + + let z_index = match params.z_index_property_name.as_deref() { + Some("updated") => item + .properties + .updated + .as_deref() + .and_then(|updated| chrono::DateTime::parse_from_rfc3339(updated).ok()) + .map_or_else(|| datetime.timestamp_millis(), |dt| dt.timestamp_millis()), + _ => 0, + }; + + for dataset in &provider_def.datasets { + for (band_idx, band_def) in dataset.bands.iter().enumerate() { + let Some((_asset_key, asset)) = item + .assets + .iter() + .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title)) + else { + continue; + }; + + // Check data type matches + if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) + && asset_dt != dataset.data_type + { + continue; + } + + // Extract the item's actual EPSG code from the asset + let item_epsg = common::epsg_code_from_fields( + common::StacExtensionMajorVersion::V2, + &asset.additional_fields, + ) + .or_else(|| { + // Also try to extract from serialized properties as fallback + let props_val = serde_json::to_value(&item.properties) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(); + common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) + }); + + let Some(item_epsg) = item_epsg else { + continue; + }; + + // Determine the actual projection and dataset name for this item + let (_actual_projection, actual_dataset_name) = if dataset.projection + == SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) + { + // EPSG matches the mapping — use the dataset as-is + ( + dataset.projection, + dataset_name_for_harvest(&provider_def.collection_name, dataset), + ) + } else { + // EPSG differs — create a per-EPSG variant + let per_epsg_projection = + SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg); + + // Build a modified dataset with the item's EPSG + let per_epsg_dataset = StacProviderDataset { + projection: per_epsg_projection, + ..dataset.clone() + }; + + let dyn_name = + dataset_name_for_harvest(&provider_def.collection_name, &per_epsg_dataset); + + // Register this dynamic dataset so it gets created + dynamic_datasets + .entry(dyn_name.clone()) + .or_insert_with(|| StacProviderDataset { + projection: SpatialReference::new( + SpatialReferenceAuthority::Epsg, + item_epsg, + ), + spatial_grid: dataset.spatial_grid, + ..dataset.clone() + }); + + (per_epsg_projection, dyn_name) + }; + + let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) + else { + continue; + }; + + if (geo_transform.x_pixel_size().abs() - dataset.resolution.x).abs() > 1e-9 + || (geo_transform.y_pixel_size().abs() - dataset.resolution.y).abs() > 1e-9 + { + continue; + } + + let Some((height, width)) = common::proj_shape_from_fields(&asset.additional_fields) + else { + continue; + }; + + let Some(rasterband_channel) = + common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref()) + else { + continue; + }; + + let grid_bounds = GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .context("Failed to create grid bounds")?; + + let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); + + let Some(file_path) = common::gdal_file_path(&asset.href) else { + continue; + }; + + let gdal_config_options = common::gdal_config_options_for_file_path( + &file_path, + provider_def.s3_config.as_ref(), + ); + + let tile = AddDatasetTile { + time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) + .context("Failed to create time interval")? + .into(), + spatial_partition: spatial_partition.into(), + band: band_idx as u32, + z_index, + params: GdalDatasetParameters { + file_path, + rasterband_channel, + geo_transform: geo_transform.into(), + width, + height, + file_not_found_handling: + crate::api::model::operators::FileNotFoundHandling::Error, + no_data_value: params.no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: gdal_config_options.map(|opts| { + opts.into_iter() + .map(|(k, v)| GdalConfigOption::from((k, v))) + .collect() + }), + allow_alphaband_as_mask: false, + }, + }; + + tiles_by_dataset + .entry(actual_dataset_name) + .or_default() + .push(tile); + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Dataset and Layer Creation (Harvest) +// --------------------------------------------------------------------------- + +fn dataset_name_for_harvest(collection_name: &str, dataset: &StacProviderDataset) -> String { + let cleaned_name: String = collection_name + .chars() + .map(|c| { + if geoengine_datatypes::dataset::is_invalid_name_char(c) { + '_' + } else { + c + } + }) + .collect(); + + let resolution_str = format!("{}", dataset.resolution.x); + let clean_resolution: String = resolution_str + .chars() + .map(|c| { + if geoengine_datatypes::dataset::is_invalid_name_char(c) { + '_' + } else { + c + } + }) + .collect(); + + format!( + "{}_EPSG{}_{:?}_{}", + cleaned_name, + dataset.projection.code(), + dataset.data_type, + clean_resolution + ) +} + +async fn dataset_exists_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + dataset_name: &str, +) -> Result { + let response = retry_http( + || async { + client + .get(format!("{geo_engine_url}/dataset/{dataset_name}")) + .header("Authorization", format!("Bearer {session_id}")) + .send() + .await + }, + &format!("Check dataset existence for '{dataset_name}'"), + ) + .await?; + + if response.status().is_success() { + return Ok(true); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if status == reqwest::StatusCode::BAD_REQUEST + && let Ok(error_response) = serde_json::from_str::(&body) + && error_response.error == "CannotLoadDataset" + { + return Ok(false); + } + + anyhow::bail!("Failed to check dataset '{dataset_name}': HTTP {status}: {body}"); +} + +async fn create_dataset_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + dataset_name: &str, + dataset: &StacProviderDataset, + volume_name: &str, +) -> Result<(), anyhow::Error> { + let bands: Vec = dataset + .bands + .iter() + .map(|b| RasterBandDescriptor { + name: b.band_name.clone().unwrap_or_else(|| b.asset_title.clone()), + measurement: Measurement::Unitless(UnitlessMeasurement { + r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, + }), + }) + .collect(); + + // Get GeoTransform from the spatial grid descriptor for the API + let dt_gt: GeoTransform = dataset.spatial_grid.geo_transform(); + let api_gt: crate::api::model::datatypes::GeoTransform = dt_gt.into(); + + let create_dataset_req = CreateDataset { + data_path: DataPath::Volume(VolumeName(volume_name.to_string())), + definition: DatasetDefinition { + properties: AddDataset { + name: Some( + DatasetName::from_str(dataset_name) + .map_err(|e| anyhow::anyhow!("Failed to create dataset name: {e}"))?, + ), + display_name: dataset_name.to_string(), + description: format!("{dataset_name} harvested from STAC"), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: None, + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: GdalMultiBandTypeTag::GdalMultiBandTypeTag, + result_descriptor: RasterResultDescriptor { + data_type: dataset.data_type.into(), + spatial_reference: SpatialReferenceOption::SpatialReference(dataset.projection) + .into(), + time: TimeDescriptor { + bounds: None, + dimension: TimeDimension::Regular(RegularTimeDimension { + origin: TimeInstance::from_millis_unchecked(0).into(), + step: TimeStep { + granularity: TimeGranularity::Days, + step: 1, + }, + }), + }, + spatial_grid: SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition { + geo_transform: api_gt, + grid_bounds: ApiGridBoundingBox2D { + top_left_idx: ApiGridIdx2D { x_idx: 0, y_idx: 0 }, + bottom_right_idx: ApiGridIdx2D { x_idx: 1, y_idx: 1 }, + }, + }, + descriptor: SpatialGridDescriptorState::Source, + }, + bands: RasterBandDescriptors::new(bands).context("Invalid band descriptors")?, + }, + }), + }, + }; + + let response = retry_http( + || async { + client + .post(format!("{geo_engine_url}/dataset")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&create_dataset_req) + .send() + .await + }, + &format!("Create dataset '{dataset_name}'"), + ) + .await?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Failed to create dataset '{dataset_name}': HTTP {status}: {body}"); + } + + let created_name = if let Ok(json) = response.json::().await { + json.get("datasetName") + .and_then(|v| v.as_str()) + .unwrap_or(dataset_name) + .to_string() + } else { + dataset_name.to_string() + }; + + share_dataset_api(client, geo_engine_url, session_id, &created_name).await?; + + Ok(()) +} + +async fn share_dataset_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + dataset_name: &str, +) -> Result<(), anyhow::Error> { + let permissions = vec![ + PermissionRequest { + resource: Resource::Dataset(DatasetResource { + id: DatasetName::new(None, dataset_name.to_string()), + r#type: DatasetResourceTypeTag::DatasetResourceTypeTag, + }), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: Resource::Dataset(DatasetResource { + id: DatasetName::new(None, dataset_name.to_string()), + r#type: DatasetResourceTypeTag::DatasetResourceTypeTag, + }), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + retry_http( + || async { + client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(permission) + .send() + .await + }, + &format!("Add permission for dataset '{dataset_name}'"), + ) + .await?; + } + + Ok(()) +} + +async fn create_harvest_layer_collections( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + provider_def: &StacDataProviderDefinition, + created_datasets: &[(usize, StacProviderDataset)], + params: &StacHarvest, +) -> Result<(), anyhow::Error> { + let root_collection_id = create_layer_collection_api( + client, + geo_engine_url, + session_id, + &LayerCollectionId(INTERNAL_LAYER_DB_ROOT_COLLECTION_ID.to_string()), + &provider_def.collection_name, + &format!( + "{} datasets harvested from STAC", + provider_def.collection_name + ), + params, + ) + .await?; + + let temp_collection_id = create_layer_collection_api( + client, + geo_engine_url, + session_id, + &root_collection_id, + "_layers", + "All dataset layers (internal)", + params, + ) + .await?; + + for (_idx, dataset) in created_datasets { + let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + let layer_name = format!( + "EPSG:{} {:?} {}m", + dataset.projection.authority(), + dataset.data_type, + dataset.resolution.x + ); + + let add_layer = AddLayer { + name: layer_name.clone(), + description: format!("Dataset: {dataset_name}"), + workflow: Workflow::Legacy { + operator: TypedOperator::Raster( + MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(NamedData { + namespace: None, + provider: None, + name: dataset_name.clone(), + }), + } + .boxed(), + ), + }, + symbology: None, + properties: vec![], + metadata: Default::default(), + }; + + let response: IdResponse = retry_http( + || async { + client + .post(format!( + "{geo_engine_url}/layerDb/collections/{temp_collection_id}/layers" + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&add_layer) + .send() + .await? + .json() + .await + }, + &format!("Create layer '{layer_name}'"), + ) + .await?; + + share_layer_api(client, geo_engine_url, session_id, &response.id).await?; + } + + Ok(()) +} + +async fn create_layer_collection_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + parent_id: &LayerCollectionId, + name: &str, + description: &str, + params: &StacHarvest, +) -> Result { + if let Some(existing_id) = + find_child_collection_by_name(client, geo_engine_url, session_id, parent_id, name).await? + { + if params.verbose { + info!("Found existing layer collection '{name}'"); + } + return Ok(existing_id); + } + + let add_collection = AddLayerCollection { + name: name.to_string(), + description: description.to_string(), + properties: vec![], + }; + + let response: IdResponse = retry_http( + || async { + client + .post(format!( + "{geo_engine_url}/layerDb/collections/{parent_id}/collections" + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&add_collection) + .send() + .await? + .json() + .await + }, + &format!("Create layer collection '{name}'"), + ) + .await?; + + share_layer_collection_api(client, geo_engine_url, session_id, &response.id).await?; + + Ok(response.id) +} + +async fn find_child_collection_by_name( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + parent_id: &LayerCollectionId, + child_name: &str, +) -> Result, anyhow::Error> { + let mut offset: u32 = 0; + let limit: u32 = 20; + + loop { + let response: LayerCollection = retry_http( + || async { + client + .get(format!( + "{geo_engine_url}/layers/collections/{INTERNAL_PROVIDER_ID}/{parent_id}" + )) + .query(&[("offset", offset), ("limit", limit)]) + .header("Authorization", format!("Bearer {session_id}")) + .send() + .await? + .json() + .await + }, + &format!("List child collections of {parent_id}"), + ) + .await?; + + for item in &response.items { + if let CollectionItem::Collection(collection) = item + && collection.name == child_name + { + return Ok(Some(collection.id.collection_id.clone())); + } + } + + if response.items.len() < limit as usize { + return Ok(None); + } + + offset += limit; + } +} + +async fn share_layer_collection_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + collection_id: &LayerCollectionId, +) -> Result<(), anyhow::Error> { + let permissions = vec![ + PermissionRequest { + resource: Resource::LayerCollection(LayerCollectionResource { + id: collection_id.clone(), + r#type: LayerCollectionResourceTypeTag::LayerCollectionResourceTypeTag, + }), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: Resource::LayerCollection(LayerCollectionResource { + id: collection_id.clone(), + r#type: LayerCollectionResourceTypeTag::LayerCollectionResourceTypeTag, + }), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + retry_http( + || async { + client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(permission) + .send() + .await + }, + &format!("Share collection with role {}", permission.role_id), + ) + .await?; + } + + Ok(()) +} + +async fn share_layer_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + layer_id: &LayerId, +) -> Result<(), anyhow::Error> { + let permissions = vec![ + PermissionRequest { + resource: Resource::Layer(LayerResource { + id: layer_id.clone(), + r#type: LayerResourceTypeTag::LayerResourceTypeTag, + }), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: Resource::Layer(LayerResource { + id: layer_id.clone(), + r#type: LayerResourceTypeTag::LayerResourceTypeTag, + }), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + retry_http( + || async { + client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(permission) + .send() + .await + }, + &format!("Share layer with role {}", permission.role_id), + ) + .await?; + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Pagination +// --------------------------------------------------------------------------- + +const MAX_RETRIES: u32 = 10; +const INITIAL_RETRY_DELAY_MS: u64 = 1000; + +#[derive(Debug, Clone)] +enum QueryState { + FirstPage { + query_url: String, + query_params: Vec<(String, String)>, + }, + NextPage { + next_url: String, + }, + Finished, +} + +fn create_page_stream( + initial_query_state: QueryState, + client: reqwest::Client, + _verbose: bool, + prefetch_buffer: usize, +) -> impl futures::Stream> { + let page_stream = futures::stream::unfold( + (client, initial_query_state), + move |(client, state)| async move { + if matches!(state, QueryState::Finished) { + return None; + } + + debug!("Fetching page: {state:?}"); + + let result = query_item_collection_internal(&client, &state).await; + + match result { + Ok((item_collection, new_state)) => { + if item_collection.items.is_empty() { + None + } else { + Some((Ok(item_collection), (client, new_state))) + } + } + Err(e) => { + error!("Error fetching page: {e:#}"); + Some((Err(e), (client, QueryState::Finished))) + } + } + }, + ); + page_stream + .map(|result| async move { result }) + .buffered(prefetch_buffer) +} + +async fn query_item_collection_internal( + client: &reqwest::Client, + query_state: &QueryState, +) -> Result<(stac::ItemCollection, QueryState), anyhow::Error> { + match query_state { + QueryState::FirstPage { + query_url, + query_params, + } => { + let item_collection: stac::ItemCollection = retry_http( + || async { + client + .get(query_url) + .query(&query_params) + .send() + .await? + .json() + .await + }, + "Query STAC first page", + ) + .await?; + + let new_state = item_collection + .links + .iter() + .find(|link| link.rel == "next") + .map_or(QueryState::Finished, |link| QueryState::NextPage { + next_url: link.href.clone(), + }); + + Ok((item_collection, new_state)) + } + QueryState::NextPage { next_url } => { + let item_collection: stac::ItemCollection = retry_http( + || async { client.get(next_url).send().await?.json().await }, + "Query STAC next page", + ) + .await?; + + let new_state = item_collection + .links + .iter() + .find(|link| link.rel == "next") + .map_or(QueryState::Finished, |link| QueryState::NextPage { + next_url: link.href.clone(), + }); + + Ok((item_collection, new_state)) + } + QueryState::Finished => anyhow::bail!("No more pages to query"), + } +} + +// --------------------------------------------------------------------------- +// HTTP retry helper +// --------------------------------------------------------------------------- + +async fn retry_http(mut operation: F, operation_name: &str) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + E: std::fmt::Display, +{ + let mut attempt = 0; + loop { + match operation().await { + Ok(result) => return Ok(result), + Err(err) => { + attempt += 1; + if attempt >= MAX_RETRIES { + error!("{operation_name} failed after {MAX_RETRIES} attempts: {err}"); + return Err(err); + } + let delay = Duration::from_millis(INITIAL_RETRY_DELAY_MS * 2_u64.pow(attempt - 1)); + warn!( + "{operation_name} failed (attempt {attempt}/{MAX_RETRIES}): {err}. Retrying in {delay:?}..." + ); + tokio::time::sleep(delay).await; + } + } + } +} + +// --------------------------------------------------------------------------- +// Authentication helper +// --------------------------------------------------------------------------- + +async fn login_geo_engine( + geo_engine_url: &str, + geo_engine_email: &str, + geo_engine_password: &str, +) -> Result<(reqwest::Client, String), anyhow::Error> { + let client = reqwest::Client::new(); + + let response = retry_http( + || async { + client + .post(format!("{geo_engine_url}/login")) + .header("Content-Type", "application/json") + .json(&serde_json::json!({ + "email": geo_engine_email, + "password": geo_engine_password, + })) + .send() + .await + }, + "Login to Geo Engine", + ) + .await + .context("Failed to authenticate")?; + + let json = response + .json::() + .await + .context("Failed to parse auth response")?; + + let session_id = json["id"] + .as_str() + .context("No session id in response")? + .to_string(); + + Ok((client, session_id)) +} + +fn data_type_from_asset_v1_1_0_fallback(asset: &stac::Asset) -> Option { + common::data_type_from_asset_v1_1_0(asset).or_else(|| { + asset + .additional_fields + .get("data_type") + .and_then(|v| v.as_str()) + .and_then(common::raster_data_type_from_stac_data_type_str) + }) +} + +fn format_duration(secs: u64) -> String { + if secs < 60 { + format!("{secs}s") + } else if secs < 3600 { + format!("{}m{}s", secs / 60, secs % 60) + } else { + format!("{}h{}m{}s", secs / 3600, (secs % 3600) / 60, secs % 60) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use geoengine_datatypes::primitives::SpatialResolution; + + #[test] + fn test_dataset_name_for_harvest() { + let collection = "sentinel-2-l2a"; + let dataset = StacProviderDataset { + name: "Test".to_string(), + description: String::new(), + data_type: RasterDataType::U16, + resolution: SpatialResolution::new_unchecked(10.0, 10.0), + projection: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632), + spatial_grid: geoengine_operators::engine::SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0.0, 0.0).into(), 10.0, -10.0), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])).unwrap(), + ), + bands: vec![], + }; + + let name = dataset_name_for_harvest(collection, &dataset); + assert!(name.contains("sentinel-2-l2a")); + assert!(name.contains("EPSG")); + assert!(name.contains("U16")); + } + + #[test] + fn test_process_harvest_item_produces_correct_tiles() { + let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid mapping fixture"); + + let items: stac::ItemCollection = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/code-de-harvest-test.json" + )) + .expect("valid items fixture"); + + let params = StacHarvest { + mapping: mapping.clone(), + time_start: None, + time_end: None, + bbox: None, + limit: None, + geo_engine_url: String::new(), + geo_engine_email: String::new(), + geo_engine_password: String::new(), + volume_name: String::new(), + verbose: false, + prefetch_pages: 1, + z_index_property_name: Some("updated".to_string()), + no_data_value: None, + gdal_retries: None, + filter_item_fields: true, + }; + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut dynamic_datasets: HashMap = HashMap::new(); + + // Process the first item from the fixture + let item = &items.items[0]; + + process_harvest_item_dynamic( + item, + &mapping, + &mut tiles_by_dataset, + &mut dynamic_datasets, + ¶ms, + ) + .expect("item processing should succeed"); + + // The mapping has 2 datasets (10m and 20m), the item has assets for both + assert_eq!( + tiles_by_dataset.len(), + 2, + "tiles should be produced for 2 datasets (10m and 20m)" + ); + + // Check each dataset has valid tiles + for (dataset_name, tiles) in &tiles_by_dataset { + assert!( + !tiles.is_empty(), + "dataset {dataset_name} should have tiles" + ); + for tile in tiles { + assert!(tile.band < 10, "band index should be reasonable"); + assert!( + tile.params.width > 0 && tile.params.height > 0, + "tile dimensions should be positive" + ); + assert!( + tile.params + .file_path + .to_string_lossy() + .starts_with("/vsis3/"), + "file path should be a VSI path: {}", + tile.params.file_path.display() + ); + } + } + + // Verify no dynamic (per-EPSG) datasets since all items match the mapping EPSG + assert!( + dynamic_datasets.is_empty(), + "no dynamic datasets should be needed when EPSGs match" + ); + + // 10m dataset should have 4 tiles (B02, B03, B04, B08) + let total_tiles_10m: usize = tiles_by_dataset + .iter() + .filter(|(name, _)| name.contains("10")) + .map(|(_, tiles)| tiles.len()) + .sum(); + assert_eq!( + total_tiles_10m, 4, + "first item should produce 4 tiles for 10m bands" + ); + + // 20m dataset should have 2 tiles (B11, B12) + let total_tiles_20m: usize = tiles_by_dataset + .iter() + .filter(|(name, _)| name.contains("20")) + .map(|(_, tiles)| tiles.len()) + .sum(); + assert_eq!( + total_tiles_20m, 2, + "first item should produce 2 tiles for 20m bands" + ); + } + + #[test] + fn test_process_harvest_landsat_item_produces_correct_tiles() { + let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" + )) + .expect("valid Landsat mapping fixture"); + + let items: stac::ItemCollection = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/landsat-c2-l1-harvest-test.json" + )) + .expect("valid Landsat items fixture"); + + let params = StacHarvest { + mapping: mapping.clone(), + time_start: None, + time_end: None, + bbox: None, + limit: None, + geo_engine_url: String::new(), + geo_engine_email: String::new(), + geo_engine_password: String::new(), + volume_name: String::new(), + verbose: false, + prefetch_pages: 1, + z_index_property_name: Some("updated".to_string()), + no_data_value: None, + gdal_retries: None, + filter_item_fields: true, + }; + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut dynamic_datasets: HashMap = HashMap::new(); + + let item = &items.items[0]; + + process_harvest_item_dynamic( + item, + &mapping, + &mut tiles_by_dataset, + &mut dynamic_datasets, + ¶ms, + ) + .expect("Landsat item processing should succeed"); + + // Mapping has 1 dataset (30m), the item has assets for it + assert_eq!( + tiles_by_dataset.len(), + 1, + "tiles should be produced for 1 dataset (30m)" + ); + + for (dataset_name, tiles) in &tiles_by_dataset { + assert!( + !tiles.is_empty(), + "dataset {dataset_name} should have tiles" + ); + for tile in tiles { + assert!(tile.band < 10, "band index should be reasonable"); + assert!( + tile.params.width > 0 && tile.params.height > 0, + "tile dimensions should be positive" + ); + assert!( + tile.params + .file_path + .to_string_lossy() + .starts_with("/vsis3/"), + "file path should be a VSI path: {}", + tile.params.file_path.display() + ); + } + } + + assert!( + dynamic_datasets.is_empty(), + "no dynamic datasets should be needed when EPSGs match" + ); + + // 30m dataset should have 4 tiles (Blue, Green, Red, NIR) + let total_tiles_30m: usize = tiles_by_dataset + .iter() + .filter(|(name, _)| name.contains("30")) + .map(|(_, tiles)| tiles.len()) + .sum(); + assert_eq!( + total_tiles_30m, 4, + "first item should produce 4 tiles for 30m bands" + ); + } +} diff --git a/geoengine/services/src/cli/stac_harvester/mod.rs b/geoengine/services/src/cli/stac_harvester/mod.rs new file mode 100644 index 0000000000..7ab60a16f6 --- /dev/null +++ b/geoengine/services/src/cli/stac_harvester/mod.rs @@ -0,0 +1,47 @@ +//! New STAC harvester CLI that separates mapping generation from tile harvesting. +//! +//! # Subcommands +//! +//! - `discover-mapping`: Probes a STAC collection and items API to generate a +//! `StacDataProviderDefinition` JSON that maps STAC assets to Geo Engine datasets. +//! - `harvest`: Reads a `StacDataProviderDefinition` and harvests tiles into Geo Engine, +//! creating datasets, tiles, and layer collections. +//! +//! The mapping JSON matches the format of `StacDataProviderDefinition` as used by the +//! STAC provider and the EDV bootstrap scripts. + +#![allow(clippy::print_stdout)] + +mod discover; +mod harvest; + +pub use discover::StacDiscoverMapping; +pub use harvest::StacHarvest; + +use clap::{Parser, Subcommand}; + +/// STAC harvester for Geo Engine +#[derive(Debug, Parser)] +pub struct StacHarvester { + #[clap(subcommand)] + pub command: StacHarvesterCommand, +} + +#[derive(Debug, Subcommand)] +#[allow(clippy::enum_variant_names)] +pub enum StacHarvesterCommand { + /// Probe a STAC API to auto-discover the dataset mapping + DiscoverMapping(Box), + /// Harvest tiles using a predefined dataset mapping + Harvest(Box), +} + +/// Run the STAC harvester +pub async fn stac_harvester(params: StacHarvester) -> Result<(), anyhow::Error> { + match params.command { + StacHarvesterCommand::DiscoverMapping(discover) => { + discover::discover_mapping(*discover).await + } + StacHarvesterCommand::Harvest(harvest) => harvest::harvest_tiles(*harvest).await, + } +} diff --git a/geoengine/services/src/cli/stac_import.rs b/geoengine/services/src/cli/stac_import.rs index c57c31902b..e7f0143cf1 100644 --- a/geoengine/services/src/cli/stac_import.rs +++ b/geoengine/services/src/cli/stac_import.rs @@ -1901,9 +1901,21 @@ impl DatasetKey { }) .collect::(); + let resolution_str = format!("{}", self.resolution); + let clean_resolution: String = resolution_str + .chars() + .map(|c| { + if geoengine_datatypes::dataset::is_invalid_name_char(c) { + '_' + } else { + c + } + }) + .collect(); + format!( "{}_EPSG{}_{:?}_{}", - cleaned_name, self.epsg, self.data_type, self.resolution + cleaned_name, self.epsg, self.data_type, clean_resolution ) } } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs new file mode 100644 index 0000000000..89a47f905a --- /dev/null +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -0,0 +1,984 @@ +//! Common utilities shared between the STAC provider and the STAC harvester CLI. +//! +//! This module contains functions for parsing STAC metadata (geometry, EPSG, data types, +//! band names), STAC API query helpers, and GDAL file path handling. +//! +//! All functions herein should be usable from both `loading_info.rs` (the STAC provider) +//! and `cli/stac_harvester.rs` (the new STAC harvester CLI). + +#![allow(dead_code)] + +use geoengine_datatypes::{ + raster::{GdalGeoTransform, GeoTransform, RasterDataType}, + spatial_reference::SpatialReference, +}; +use serde::Deserialize; +use std::path::PathBuf; + +use super::StacProviderS3Config; + +// --------------------------------------------------------------------------- +// STAC extension version types +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StacExtensionMajorVersion { + V1, + V2, +} + +/// Extract a `GeoTransform` from `proj:transform` in asset/collection fields. +/// +/// The STAC `proj:transform` is a 6-element array in GDAL convention: +/// `[pixel_width, rotation, origin_x, rotation, pixel_height, origin_y]`. +pub fn geo_transform_from_fields( + fields: &serde_json::Map, +) -> Option { + let proj_transform = fields.get("proj:transform")?; + let proj_transform_array = proj_transform.as_array()?; + if proj_transform_array.len() != 6 { + return None; + } + + let values: Vec = proj_transform_array + .iter() + .filter_map(serde_json::Value::as_f64) + .collect(); + if values.len() != 6 { + return None; + } + + // GDAL geo-transform: [origin_x, pixel_width, rotation, origin_y, rotation, pixel_height] + let gdal_geotransform: GdalGeoTransform = [ + values[2], // origin_x + values[0], // pixel_width + values[1], // rotation + values[5], // origin_y + values[3], // rotation + values[4], // pixel_height (negative for north-up) + ]; + Some(gdal_geotransform.into()) +} + +/// Extract `(height, width)` from `proj:shape` in asset fields. +pub fn proj_shape_from_fields( + fields: &serde_json::Map, +) -> Option<(usize, usize)> { + let proj_shape = fields.get("proj:shape")?.as_array()?; + if proj_shape.len() != 2 { + return None; + } + let height = proj_shape.first()?.as_u64()? as usize; + let width = proj_shape.get(1)?.as_u64()? as usize; + + Some((height, width)) +} + +// --------------------------------------------------------------------------- +// EPSG / projection helpers +// --------------------------------------------------------------------------- + +/// Extract EPSG code from asset fields, respecting the STAC extension version's +/// field priority (`proj:epsg` vs `proj:code`). +pub fn epsg_code_from_fields( + proj_extension_version: StacExtensionMajorVersion, + fields: &serde_json::Map, +) -> Option { + let proj_epsg = fields.get("proj:epsg").and_then(|value| { + value + .as_u64() + .map(|code| code as u32) + .or_else(|| value.as_str().and_then(|code| code.parse::().ok())) + }); + + let proj_code = fields + .get("proj:code") + .and_then(serde_json::Value::as_str) + .and_then(parse_epsg_from_proj_code); + + match proj_extension_version { + StacExtensionMajorVersion::V1 => proj_epsg.or(proj_code), + StacExtensionMajorVersion::V2 => proj_code.or(proj_epsg), + } +} + +/// Extract EPSG code from a STAC item, with multi-layered fallback between +/// additional fields and serialized properties, and fallback extension version. +pub fn epsg_code_from_item( + item: &stac::Item, + proj_extension_version: StacExtensionMajorVersion, +) -> Option { + let from_additional = + epsg_code_from_fields(proj_extension_version, &item.properties.additional_fields); + if from_additional.is_some() { + return from_additional; + } + + let properties = serde_json::to_value(item) + .ok() + .and_then(|value| value.get("properties").cloned()) + .and_then(|value| value.as_object().cloned())?; + + let from_properties = epsg_code_from_fields(proj_extension_version, &properties); + if from_properties.is_some() { + return from_properties; + } + + let fallback_version = match proj_extension_version { + StacExtensionMajorVersion::V1 => StacExtensionMajorVersion::V2, + StacExtensionMajorVersion::V2 => StacExtensionMajorVersion::V1, + }; + + epsg_code_from_fields(fallback_version, &properties) +} + +/// Parse an EPSG code from a `proj:code` string like `"EPSG:32632"` or +/// `"http://www.opengis.net/def/crs/EPSG/0/32632"`. +pub fn parse_epsg_from_proj_code(code: &str) -> Option { + if let Some(code) = code.strip_prefix("EPSG:") { + return code.parse::().ok(); + } + + // e.g. http://www.opengis.net/def/crs/EPSG/0/32632 + code.rsplit('/').next()?.parse::().ok() +} + +/// Check if the asset's `proj:code` matches the dataset's projection. +pub fn proj_code_matches_dataset( + fields: &serde_json::Map, + dataset_projection: SpatialReference, +) -> bool { + let Some(code) = fields.get("proj:code") else { + return false; + }; + + let Some(proj_code) = proj_code_as_srs_string(code) else { + return false; + }; + + proj_code == dataset_projection.to_string() +} + +/// Normalize a `proj:code` field value to an `"EPSG:nnnn"` string. +pub fn proj_code_as_srs_string(value: &serde_json::Value) -> Option { + if let Some(code_number) = value.as_u64() { + return Some(format!("EPSG:{code_number}")); + } + + let code_str = value.as_str()?.trim(); + if code_str.contains(':') { + return Some(code_str.to_ascii_uppercase()); + } + + if let Ok(code_number) = code_str.parse::() { + return Some(format!("EPSG:{code_number}")); + } + + None +} + +// --------------------------------------------------------------------------- +// Data type conversion helpers +// --------------------------------------------------------------------------- + +/// Map a `stac_extensions::raster::DataType` to a Geo Engine `RasterDataType`. +pub fn raster_data_type_from_stac_data_type( + data_type: &stac_extensions::raster::DataType, +) -> Option { + match data_type { + stac_extensions::raster::DataType::UInt8 => Some(RasterDataType::U8), + stac_extensions::raster::DataType::UInt16 => Some(RasterDataType::U16), + stac_extensions::raster::DataType::UInt32 => Some(RasterDataType::U32), + stac_extensions::raster::DataType::Int16 => Some(RasterDataType::I16), + stac_extensions::raster::DataType::Int32 => Some(RasterDataType::I32), + stac_extensions::raster::DataType::Float32 => Some(RasterDataType::F32), + stac_extensions::raster::DataType::Float64 => Some(RasterDataType::F64), + _ => None, + } +} + +/// Map a STAC data type string (e.g. `"uint16"`, `"float32"`) to a `RasterDataType`. +pub fn raster_data_type_from_stac_data_type_str(data_type_str: &str) -> Option { + match data_type_str.to_lowercase().as_str() { + "uint8" => Some(RasterDataType::U8), + "uint16" => Some(RasterDataType::U16), + "uint32" => Some(RasterDataType::U32), + "int16" => Some(RasterDataType::I16), + "int32" => Some(RasterDataType::I32), + "float32" => Some(RasterDataType::F32), + "float64" => Some(RasterDataType::F64), + _ => None, + } +} + +/// Extract data type from a STAC 1.1.0 asset (common metadata `data_type` field). +pub fn data_type_from_asset_v1_1_0(asset: &stac::Asset) -> Option { + asset + .data_type + .as_ref() + .and_then(raster_data_type_from_stac_data_type) +} + +// --------------------------------------------------------------------------- +// File path helpers +// --------------------------------------------------------------------------- + +/// Convert a STAC asset href (HTTP or S3 URL) to a GDAL VSI path. +/// +/// - `http://...` → `/vsicurl/http://...` +/// - `s3://bucket/key` → `/vsis3/bucket/key` +pub fn gdal_file_path(href: &str) -> Option { + if href.starts_with("http") { + return Some(PathBuf::from(format!("/vsicurl/{href}"))); + } + + href.strip_prefix("s3://") + .map(|s3_path| PathBuf::from(format!("/vsis3/{s3_path}"))) +} + +// --------------------------------------------------------------------------- +// Band processing helpers +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +pub struct EoBand { + pub name: String, + #[serde(default)] + pub common_name: Option, +} + +/// Map a GDAL raster band channel index for a dataset band within an asset. +/// +/// If the asset has no `bands` metadata, returns channel 1 (single-band asset). +/// If the asset has bands, matches by `band_name` against asset band names. +/// If the asset has exactly one band and no `band_name` is required, returns +/// channel 1 (single-band asset treated the same as no band metadata). +/// Returns `None` if the required band is not found. +pub fn rasterband_channel_for_dataset_band( + asset: &stac::Asset, + required_band_name: Option<&str>, +) -> Option { + if asset.bands.is_empty() || (asset.bands.len() == 1 && required_band_name.is_none()) { + if required_band_name.is_some() && asset.bands.is_empty() { + tracing::warn!( + "STAC asset with href {} does not include bands, but dataset configuration requires a band name. Skipping asset.", + asset.href + ); + return None; + } + + return Some(1); + } + + let Some(required_band_name) = required_band_name else { + tracing::warn!( + "STAC asset with href {} includes {} bands, but dataset configuration does not specify a band name. Skipping asset.", + asset.href, + asset.bands.len() + ); + return None; + }; + + let Some(asset_band_idx) = asset + .bands + .iter() + .position(|asset_band| asset_band.name.as_deref() == Some(required_band_name)) + else { + tracing::debug!( + "Skipping asset with href {} due to missing required band {}", + asset.href, + required_band_name + ); + return None; + }; + + Some(asset_band_idx + 1) +} + +/// Derive band names from a STAC 1.1.0 `Asset`, using the `bands` field. +pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result, String> { + let asset_title = asset + .title + .as_deref() + .ok_or_else(|| "Missing title in asset metadata".to_string())?; + + let bands = &asset.bands; + + if bands.is_empty() { + return Ok(vec![asset_title.to_string()]); + } + + if bands.len() == 1 { + return Ok(vec![asset_title.to_string()]); + } + + let mut names = Vec::new(); + for band in bands { + let Some(band_name) = &band.name else { + return Err("Band is missing name for multi-band asset".to_string()); + }; + names.push(format!("{asset_title} [{band_name}]")); + } + + Ok(names) +} + +/// Derive band names from a STAC 1.1.0 `ItemAsset`, using the `bands` additional field. +pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result, String> { + let asset_title = asset + .title + .as_deref() + .ok_or_else(|| "Missing title in asset metadata".to_string())?; + + let band_names = asset + .additional_fields + .get("bands") + .and_then(serde_json::Value::as_array); + + let Some(bands) = band_names else { + return Ok(vec![asset_title.to_string()]); + }; + + if bands.is_empty() { + return Ok(vec![asset_title.to_string()]); + } + + if bands.len() == 1 { + return Ok(vec![asset_title.to_string()]); + } + + let mut names = Vec::new(); + for band in bands { + let band_name = band + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| "Band is missing name for multi-band asset".to_string())?; + names.push(format!("{asset_title} [{band_name}]")); + } + + Ok(names) +} + +/// Normalize a label string: trim, lowercase, join whitespace-separated words with underscores. +pub fn normalize_label(value: &str) -> String { + value + .trim() + .to_lowercase() + .split_whitespace() + .collect::>() + .join("_") +} + +/// Derive a fallback band label from an asset title. +/// +/// Prefers concise acronym-like labels in parentheses, e.g. `"Scene classification map (SCL)"` → `"scl"`. +/// Falls back to a normalized version of the full title. +pub fn title_fallback_label(title: Option<&str>) -> String { + if let Some(title) = title { + // Prefer concise acronym-like labels in parentheses + if let (Some(start), Some(end)) = (title.rfind('('), title.rfind(')')) + && start < end + { + let short = title[start + 1..end].trim(); + if !short.is_empty() + && short.len() <= 32 + && short + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return short.to_lowercase(); + } + } + + let normalized = normalize_label(title); + if !normalized.is_empty() { + return normalized; + } + } + + "band".to_string() +} + +/// Map red-edge variant names for Sentinel-2 bands B05/B06/B07 from metadata. +pub fn rededge_variant_from_metadata(eo_name: &str, title: Option<&str>) -> Option<&'static str> { + let eo = eo_name.to_lowercase(); + let title_lower = title.map(str::to_lowercase).unwrap_or_default(); + + if eo.contains("b05") + || eo.contains("band_5") + || title_lower.contains("band 5") + || title_lower.contains("b05") + { + return Some("rededge1"); + } + if eo.contains("b06") + || eo.contains("band_6") + || title_lower.contains("band 6") + || title_lower.contains("b06") + { + return Some("rededge2"); + } + if eo.contains("b07") + || eo.contains("band_7") + || title_lower.contains("band 7") + || title_lower.contains("b07") + { + return Some("rededge3"); + } + + None +} + +/// Derive a stable band name from STAC 1.0.0 metadata (EO band + asset title). +pub fn v1_0_0_band_name( + title: Option<&str>, + eo_band: Option<&EoBand>, + band_count: usize, +) -> String { + let eo_name = eo_band.and_then(|band| { + let eo_name = band.name.to_lowercase(); + let common_name = band.common_name.as_ref().map(|name| name.to_lowercase()); + + match common_name.as_deref() { + // `rededge` is used for multiple Sentinel-2 bands (B05/B06/B07). + // Keep stable, unique names to avoid band collisions. + Some("rededge") => rededge_variant_from_metadata(&eo_name, title) + .map(std::string::ToString::to_string) + .or_else(|| Some(format!("rededge[{eo_name}]"))), + Some(common_name) => Some(common_name.to_string()), + None => Some(eo_name), + } + }); + + if band_count > 1 { + let asset_label = title_fallback_label(title); + let eo_name = eo_name.unwrap_or_else(|| "band".to_string()); + return format!("{asset_label}[{eo_name}]"); + } + + if let Some(eo_name) = eo_name { + return eo_name; + } + + title_fallback_label(title) +} + +// --------------------------------------------------------------------------- +// Media type helpers +// --------------------------------------------------------------------------- + +/// Check if a media type is a Cloud-Optimized `GeoTIFF`. +pub fn is_cog_media_type(media_type: Option<&str>) -> bool { + media_type == Some("image/tiff; application=geotiff; profile=cloud-optimized") +} + +/// Check if a media type is JPEG 2000. +pub fn is_jp2_media_type(media_type: Option<&str>) -> bool { + media_type == Some("image/jp2") +} + +// --------------------------------------------------------------------------- +// GDAL config options +// --------------------------------------------------------------------------- + +/// Build GDAL configuration options for `/vsis3/` paths. +/// +/// Returns the common options plus S3-specific credentials when an S3 config is provided. +pub fn gdal_config_options_for_s3( + s3_config: Option<&StacProviderS3Config>, +) -> Vec<(String, String)> { + let mut options = Vec::new(); + + if let Some(config) = s3_config { + // For old GDAL versions, the S3 endpoint may not include the protocol + options.push(("AWS_S3_ENDPOINT".to_owned(), config.endpoint.clone())); + options.push(("AWS_VIRTUAL_HOSTING".to_owned(), "FALSE".to_owned())); + + if let Some(access_key) = &config.access_key { + options.push(("AWS_ACCESS_KEY_ID".to_owned(), access_key.clone())); + } + + if let Some(secret_key) = &config.secret_key { + options.push(("AWS_SECRET_ACCESS_KEY".to_owned(), secret_key.clone())); + } + } + + options +} + +/// Build GDAL configuration options for a VSI file path, including common CURL/S3 options. +pub fn gdal_config_options_for_file_path( + file_path: &std::path::Path, + s3_config: Option<&StacProviderS3Config>, +) -> Option> { + let file_path_str = file_path.to_string_lossy(); + let is_vsi_s3 = file_path_str.starts_with("/vsis3/"); + let is_vsi_curl = file_path_str.starts_with("/vsicurl/"); + + if !is_vsi_s3 && !is_vsi_curl { + return None; + } + + let mut options = vec![ + ( + "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), + "EMPTY_DIR".to_owned(), + ), + ( + "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), + ".tif,.tiff,.jp2".to_owned(), + ), + ]; + + if is_vsi_s3 { + options.extend(gdal_config_options_for_s3(s3_config)); + } + + Some(options) +} + +#[cfg(test)] +mod tests { + use super::*; + use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority}; + + // ----------------------------------------------------------------------- + // geo_transform_from_fields + // ----------------------------------------------------------------------- + + #[test] + fn test_geo_transform_from_fields_standard() { + let mut fields = serde_json::Map::new(); + fields.insert( + "proj:transform".to_string(), + serde_json::json!([10.0, 0.0, 399_960.0, 0.0, -10.0, 5_700_000.0]), + ); + + let gt = geo_transform_from_fields(&fields).expect("should parse transform"); + assert!((gt.origin_coordinate.x - 399_960.0).abs() < 1e-9); + assert!((gt.origin_coordinate.y - 5_700_000.0).abs() < 1e-9); + assert!((gt.x_pixel_size() - 10.0).abs() < 1e-9); + assert!((gt.y_pixel_size() - (-10.0)).abs() < 1e-9); + } + + #[test] + fn test_geo_transform_from_fields_missing() { + let fields = serde_json::Map::new(); + assert!(geo_transform_from_fields(&fields).is_none()); + } + + #[test] + fn test_geo_transform_from_fields_wrong_length() { + let mut fields = serde_json::Map::new(); + fields.insert( + "proj:transform".to_string(), + serde_json::json!([1.0, 2.0, 3.0]), + ); + assert!(geo_transform_from_fields(&fields).is_none()); + } + + // ----------------------------------------------------------------------- + // proj_shape_from_fields + // ----------------------------------------------------------------------- + + #[test] + fn test_proj_shape_from_fields_standard() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:shape".to_string(), serde_json::json!([10980, 10980])); + let (height, width) = proj_shape_from_fields(&fields).expect("should parse shape"); + assert_eq!(height, 10_980); + assert_eq!(width, 10_980); + } + + #[test] + fn test_proj_shape_from_fields_missing() { + let fields = serde_json::Map::new(); + assert!(proj_shape_from_fields(&fields).is_none()); + } + + // ----------------------------------------------------------------------- + // raster_data_type_from_stac_data_type + // ----------------------------------------------------------------------- + + #[test] + fn test_raster_data_type_from_stac_data_type_all() { + use stac_extensions::raster::DataType; + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::UInt8), + Some(RasterDataType::U8) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::UInt16), + Some(RasterDataType::U16) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::UInt32), + Some(RasterDataType::U32) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Int16), + Some(RasterDataType::I16) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Int32), + Some(RasterDataType::I32) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Float32), + Some(RasterDataType::F32) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Float64), + Some(RasterDataType::F64) + ); + } + + #[test] + fn test_raster_data_type_from_stac_data_type_unknown() { + use stac_extensions::raster::DataType; + assert_eq!(raster_data_type_from_stac_data_type(&DataType::Int8), None); + } + + // ----------------------------------------------------------------------- + // raster_data_type_from_stac_data_type_str + // ----------------------------------------------------------------------- + + #[test] + fn test_raster_data_type_from_stac_data_type_str_all() { + assert_eq!( + raster_data_type_from_stac_data_type_str("uint8"), + Some(RasterDataType::U8) + ); + assert_eq!( + raster_data_type_from_stac_data_type_str("uint16"), + Some(RasterDataType::U16) + ); + assert_eq!( + raster_data_type_from_stac_data_type_str("float32"), + Some(RasterDataType::F32) + ); + assert_eq!( + raster_data_type_from_stac_data_type_str("UINT16"), + Some(RasterDataType::U16) + ); + assert_eq!(raster_data_type_from_stac_data_type_str("unknown"), None); + } + + // ----------------------------------------------------------------------- + // gdal_file_path + // ----------------------------------------------------------------------- + + #[test] + fn test_gdal_file_path_http() { + let path = gdal_file_path("https://example.com/file.tif").expect("should parse"); + assert_eq!(path, PathBuf::from("/vsicurl/https://example.com/file.tif")); + } + + #[test] + fn test_gdal_file_path_s3() { + let path = gdal_file_path("s3://bucket/key/file.tif").expect("should parse"); + assert_eq!(path, PathBuf::from("/vsis3/bucket/key/file.tif")); + } + + #[test] + fn test_gdal_file_path_unsupported() { + assert!(gdal_file_path("/local/path.tif").is_none()); + } + + // ----------------------------------------------------------------------- + // parse_epsg_from_proj_code + // ----------------------------------------------------------------------- + + #[test] + fn test_parse_epsg_from_proj_code_epsg_prefix() { + assert_eq!(parse_epsg_from_proj_code("EPSG:32632"), Some(32632)); + } + + #[test] + fn test_parse_epsg_from_proj_code_url() { + assert_eq!( + parse_epsg_from_proj_code("http://www.opengis.net/def/crs/EPSG/0/32632"), + Some(32632) + ); + } + + #[test] + fn test_parse_epsg_from_proj_code_invalid() { + assert!(parse_epsg_from_proj_code("invalid").is_none()); + } + + // ----------------------------------------------------------------------- + // proj_code_as_srs_string + // ----------------------------------------------------------------------- + + #[test] + fn test_proj_code_as_srs_string_number() { + assert_eq!( + proj_code_as_srs_string(&serde_json::json!(32632)), + Some("EPSG:32632".to_string()) + ); + } + + #[test] + fn test_proj_code_as_srs_string_epsg_format() { + assert_eq!( + proj_code_as_srs_string(&serde_json::json!("EPSG:32632")), + Some("EPSG:32632".to_string()) + ); + } + + #[test] + fn test_proj_code_as_srs_string_lowercase() { + assert_eq!( + proj_code_as_srs_string(&serde_json::json!("epsg:32632")), + Some("EPSG:32632".to_string()) + ); + } + + // ----------------------------------------------------------------------- + // proj_code_matches_dataset + // ----------------------------------------------------------------------- + + #[test] + fn test_proj_code_matches_dataset_matching() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32632")); + let srs = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632); + assert!(proj_code_matches_dataset(&fields, srs)); + } + + #[test] + fn test_proj_code_matches_dataset_not_matching() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32633")); + let srs = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632); + assert!(!proj_code_matches_dataset(&fields, srs)); + } + + #[test] + fn test_proj_code_matches_dataset_missing() { + let fields = serde_json::Map::new(); + let srs = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632); + assert!(!proj_code_matches_dataset(&fields, srs)); + } + + // ----------------------------------------------------------------------- + // epsg_code_from_fields + // ----------------------------------------------------------------------- + + #[test] + fn test_epsg_code_from_fields_v1_epsg() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:epsg".to_string(), serde_json::json!(32632)); + assert_eq!( + epsg_code_from_fields(StacExtensionMajorVersion::V1, &fields), + Some(32632) + ); + } + + #[test] + fn test_epsg_code_from_fields_v1_code() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32632")); + // V1 prefers proj:epsg over proj:code + assert_eq!( + epsg_code_from_fields(StacExtensionMajorVersion::V1, &fields), + Some(32632) + ); + } + + #[test] + fn test_epsg_code_from_fields_v2_code() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32632")); + // V2 prefers proj:code over proj:epsg + assert_eq!( + epsg_code_from_fields(StacExtensionMajorVersion::V2, &fields), + Some(32632) + ); + } + + // ----------------------------------------------------------------------- + // normalize_label / title_fallback_label + // ----------------------------------------------------------------------- + + #[test] + fn test_normalize_label() { + assert_eq!( + normalize_label("Scene classification map"), + "scene_classification_map" + ); + assert_eq!(normalize_label(" Blue band "), "blue_band"); + } + + #[test] + fn test_title_fallback_label_parentheses() { + assert_eq!( + title_fallback_label(Some("Scene classification map (SCL)")), + "scl" + ); + } + + #[test] + fn test_title_fallback_label_no_parentheses() { + assert_eq!( + title_fallback_label(Some("Blue (band 2) - 10m")), + "blue_(band_2)_-_10m" + ); + } + + #[test] + fn test_title_fallback_label_none() { + assert_eq!(title_fallback_label(None), "band"); + } + + // ----------------------------------------------------------------------- + // band_names_from_asset_v1_1_0 + // ----------------------------------------------------------------------- + + #[test] + fn test_band_names_from_asset_v1_1_0_no_bands() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "title": "My Band" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + let names = band_names_from_asset_v1_1_0(&asset).expect("should succeed"); + assert_eq!(names, vec!["My Band"]); + } + + #[test] + fn test_band_names_from_asset_v1_1_0_single_band() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "title": "My Asset", + "bands": [{"name": "B01"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + let names = band_names_from_asset_v1_1_0(&asset).expect("should succeed"); + assert_eq!(names, vec!["My Asset"]); + } + + #[test] + fn test_band_names_from_asset_v1_1_0_multi_band() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "title": "Sentinel-2", + "bands": [{"name": "B04"}, {"name": "B03"}, {"name": "B02"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + let names = band_names_from_asset_v1_1_0(&asset).expect("should succeed"); + assert_eq!( + names, + vec!["Sentinel-2 [B04]", "Sentinel-2 [B03]", "Sentinel-2 [B02]",] + ); + } + + // ----------------------------------------------------------------------- + // rasterband_channel_for_dataset_band + // ----------------------------------------------------------------------- + + #[test] + fn test_rasterband_channel_no_bands_no_required() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!(rasterband_channel_for_dataset_band(&asset, None), Some(1)); + } + + #[test] + fn test_rasterband_channel_no_bands_with_required() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B04")), + None + ); + } + + #[test] + fn test_rasterband_channel_with_bands_matching() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "bands": [{"name": "B04"}, {"name": "B03"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B04")), + Some(1) + ); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B03")), + Some(2) + ); + } + + #[test] + fn test_rasterband_channel_with_bands_not_matching() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "bands": [{"name": "B04"}, {"name": "B03"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B08")), + None + ); + } + + // ----------------------------------------------------------------------- + // gdal_config_options + // ----------------------------------------------------------------------- + + #[test] + fn test_gdal_config_options_for_s3_empty() { + let options = gdal_config_options_for_s3(None); + assert!(options.is_empty()); + } + + #[test] + fn test_gdal_config_options_for_s3_with_config() { + let config = StacProviderS3Config { + endpoint: "eodata.example.com".to_string(), + access_key: Some("key".to_string()), + secret_key: Some("secret".to_string()), + }; + let options = gdal_config_options_for_s3(Some(&config)); + assert!(options.contains(&( + "AWS_S3_ENDPOINT".to_string(), + "eodata.example.com".to_string() + ))); + assert!(options.contains(&("AWS_ACCESS_KEY_ID".to_string(), "key".to_string()))); + assert!(options.contains(&("AWS_SECRET_ACCESS_KEY".to_string(), "secret".to_string()))); + assert!(options.contains(&("AWS_VIRTUAL_HOSTING".to_string(), "FALSE".to_string()))); + } + + // ----------------------------------------------------------------------- + // data_type_from_asset_v1_1_0 + // ----------------------------------------------------------------------- + + #[test] + fn test_data_type_from_asset_v1_1_0_uint16() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "data_type": "uint16" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + data_type_from_asset_v1_1_0(&asset), + Some(RasterDataType::U16) + ); + } + + #[test] + fn test_data_type_from_asset_v1_1_0_missing() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!(data_type_from_asset_v1_1_0(&asset), None); + } +} diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 43214e0916..56264daece 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -1,3 +1,4 @@ +use super::common; use super::{StacDataProvider, StacProviderDataset, StacProviderS3Config, cache::StacQueryCache}; use crate::error::Result; use crate::util::join_base_url_and_path; @@ -11,7 +12,7 @@ use geoengine_datatypes::primitives::{ AxisAlignedRectangle, CacheHint, RasterQueryRectangle, TimeDimension, TimeInstance, TimeInterval, TryRegularTimeFillIterExt, VectorQueryRectangle, }; -use geoengine_datatypes::raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}; +use geoengine_datatypes::raster::{GridBoundingBox2D, GridIdx2D}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterBandDescriptors, RasterResultDescriptor, TimeDescriptor, @@ -23,9 +24,7 @@ use geoengine_operators::source::{ GdalRetryOptions, MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, OgrSourceDataset, TileFile, }; -use serde_json::Value; use stac::Item; -use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use tracing::debug; @@ -454,15 +453,16 @@ impl StacMultiBandMetaData { z_index: i64, files: &mut Vec, ) -> Result<()> { - if data_type_from_asset_v1_1_0(asset) != Some(self.dataset.data_type) { + if common::data_type_from_asset_v1_1_0(asset) != Some(self.dataset.data_type) { return Ok(()); } - if !proj_code_matches_dataset(&asset.additional_fields, self.dataset.projection) { + if !common::proj_code_matches_dataset(&asset.additional_fields, self.dataset.projection) { return Ok(()); } - let Some(geo_transform) = geo_transform_from_fields(&asset.additional_fields) else { + let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) + else { tracing::warn!( "Skipping asset with href {} due to missing geo transform", asset.href @@ -470,7 +470,7 @@ impl StacMultiBandMetaData { return Ok(()); }; - let Some((height, width)) = proj_shape_from_fields(&asset.additional_fields) else { + let Some((height, width)) = common::proj_shape_from_fields(&asset.additional_fields) else { tracing::warn!( "Skipping asset with href {} due to missing projection shape", asset.href @@ -499,19 +499,21 @@ impl StacMultiBandMetaData { .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - let file_path = gdal_file_path(&asset.href) + let file_path = common::gdal_file_path(&asset.href) .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)?; - let gdal_config_options = self.gdal_config_options_for_file_path(&file_path); + let gdal_config_options = + common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); for (dataset_band_idx, dataset_band) in self.dataset.bands.iter().enumerate() { if dataset_band.asset_title != asset_title { continue; } - let Some(rasterband_channel) = - Self::rasterband_channel_for_dataset_band(asset, dataset_band.band_name.as_deref()) - else { + let Some(rasterband_channel) = common::rasterband_channel_for_dataset_band( + asset, + dataset_band.band_name.as_deref(), + ) else { continue; }; @@ -1142,4 +1144,70 @@ mod tests { let _result_descriptor = initialized.result_descriptor(); // If we get here, the operator initialized successfully } + + /// Test that a discover-generated mapping JSON can be used directly as a + /// `StacDataProvider`, validating the mapping format works for both the + /// harvester and the runtime provider. + #[crate::ge_context::test] + async fn mapping_from_discover_works_as_stacdataprovider(app_ctx: PostgresContext) { + // Load the discover-generated mapping JSON + let mut provider_def: crate::datasets::external::stac::StacDataProviderDefinition = + serde_json::from_str(include_str!( + "../../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid discover mapping fixture"); + + // Use a placeholder URL (no actual HTTP calls needed for meta_data registration) + provider_def.api_url = "https://stac.test/v1".to_owned(); + provider_def.id = DataProviderId::new(); + + let admin_session = admin_login(&app_ctx).await; + let admin_ctx = app_ctx.session_context(admin_session); + + admin_ctx + .db() + .add_layer_provider(provider_def.clone().into()) + .await + .unwrap(); + + let provider = admin_ctx + .db() + .load_layer_provider(provider_def.id) + .await + .unwrap(); + + // Verify each dataset from the discover-generated mapping can be + // resolved via meta_data (no HTTP calls needed at this stage) + for dataset in &provider_def.datasets { + let epsg_code = dataset.projection.code(); + let data_type_str = format!("{:?}", dataset.data_type).to_lowercase(); + let resolution = dataset.resolution.x as u32; + let stable_id = format!("epsg{epsg_code}_{data_type_str}_{resolution}"); + + let layer_id = geoengine_datatypes::dataset::LayerId(format!("dataset/{stable_id}")); + let data_id: DataId = ExternalDataId { + provider_id: provider_def.id, + layer_id, + } + .into(); + + let meta_result: Result< + Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + >, + geoengine_operators::error::Error, + > = MetaDataProvider::meta_data(provider.as_ref(), &data_id).await; + + assert!( + meta_result.is_ok(), + "meta_data should succeed for dataset '{}' (stable_id: {})", + dataset.name, + stable_id + ); + } + } } diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index f10068503e..161c32fb5b 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -14,6 +14,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; mod cache; +pub(crate) mod common; mod listing; mod loading_info; diff --git a/geoengine/test_data/stac_responses/collections/code-de-minimal.json b/geoengine/test_data/stac_responses/collections/code-de-minimal.json new file mode 100644 index 0000000000..5729100a9e --- /dev/null +++ b/geoengine/test_data/stac_responses/collections/code-de-minimal.json @@ -0,0 +1,71 @@ +{ + "id": "sentinel-2-l2a", + "description": "Sentinel-2 Level-2A", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "links": [ + { + "rel": "items", + "type": "application/geo+json", + "href": "https://stac.code-de.org/v1/collections/sentinel-2-l2a/items" + } + ], + "summaries": { + "gsd": [10, 20], + "platform": ["sentinel-2a", "sentinel-2b"] + }, + "item_assets": { + "B02_10m": { + "type": "image/jp2", + "title": "Blue (band 2) - 10m", + "bands": [{ "name": "B02", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B03_10m": { + "type": "image/jp2", + "title": "Green (band 3) - 10m", + "bands": [{ "name": "B03", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B04_10m": { + "type": "image/jp2", + "title": "Red (band 4) - 10m", + "bands": [{ "name": "B04", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B08_10m": { + "type": "image/jp2", + "title": "NIR 1 (band 8) - 10m", + "bands": [{ "name": "B08", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B11_20m": { + "type": "image/jp2", + "title": "SWIR 1 (band 11) - 20m", + "bands": [{ "name": "B11", "eo:common_name": "swir16" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490] + }, + "B12_20m": { + "type": "image/jp2", + "title": "SWIR 2 (band 12) - 20m", + "bands": [{ "name": "B12", "eo:common_name": "swir22" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490] + } + } +} diff --git a/geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json b/geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json new file mode 100644 index 0000000000..b56a7a61a3 --- /dev/null +++ b/geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json @@ -0,0 +1,55 @@ +{ + "id": "landsat-c2-l1", + "description": "Landsat Collection 2 Level-1", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "links": [ + { + "rel": "items", + "type": "application/geo+json", + "href": "https://stac.code-de.org/v1/collections/landsat-c2-l1/items" + } + ], + "summaries": { + "gsd": [30], + "platform": ["landsat-8", "landsat-9"] + }, + "item_assets": { + "B4_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red (band 4) - 30m", + "bands": [{ "name": "B4", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + }, + "B5_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR (band 5) - 30m", + "bands": [{ "name": "B5", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + }, + "B2_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Blue (band 2) - 30m", + "bands": [{ "name": "B2", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + }, + "B3_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Green (band 3) - 30m", + "bands": [{ "name": "B3", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + } + } +} diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json new file mode 100644 index 0000000000..0e49bcb127 --- /dev/null +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -0,0 +1,69 @@ +{ + "name": "sentinel-2-l2a from STAC", + "id": "00000000-0000-0000-0000-000000000000", + "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.test/v1", + "priority": 50, + "apiUrl": "https://stac.test/v1", + "collectionName": "sentinel-2-l2a", + "s3Config": null, + "timeDimension": { + "regular": { + "origin": 0, + "step": { "granularity": "days", "step": 1 } + } + }, + "datasets": [ + { + "name": "sentinel-2-l2a EPSG:32632 U16 10m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { "x": 10.0, "y": 10.0 }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "xPixelSize": 10.0, + "yPixelSize": -10.0 + }, + "gridBounds": { + "min": [0, 0], + "max": [10979, 10979] + } + }, + "state": "source" + }, + "bands": [ + { "asset_title": "Blue (band 2) - 10m", "band_name": null }, + { "asset_title": "Green (band 3) - 10m", "band_name": null }, + { "asset_title": "NIR 1 (band 8) - 10m", "band_name": null }, + { "asset_title": "Red (band 4) - 10m", "band_name": null } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U16 20m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { "x": 20.0, "y": 20.0 }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "xPixelSize": 20.0, + "yPixelSize": -20.0 + }, + "gridBounds": { + "min": [0, 0], + "max": [5489, 5489] + } + }, + "state": "source" + }, + "bands": [ + { "asset_title": "SWIR 1 (band 11) - 20m", "band_name": null }, + { "asset_title": "SWIR 2 (band 12) - 20m", "band_name": null } + ] + } + ] +} diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json new file mode 100644 index 0000000000..05713a7a10 --- /dev/null +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -0,0 +1,44 @@ +{ + "name": "landsat-c2-l1 from STAC", + "id": "00000000-0000-0000-0000-000000000000", + "description": "Auto-discovered mapping for STAC collection 'landsat-c2-l1' at https://stac.test/v1", + "priority": 50, + "apiUrl": "https://stac.test/v1", + "collectionName": "landsat-c2-l1", + "s3Config": null, + "timeDimension": { + "regular": { + "origin": 0, + "step": { "granularity": "days", "step": 1 } + } + }, + "datasets": [ + { + "name": "landsat-c2-l1 EPSG:32632 U16 30m", + "description": "Auto-discovered from STAC collection 'landsat-c2-l1'", + "data_type": "U16", + "resolution": { "x": 30.0, "y": 30.0 }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { "x": 399960.0, "y": 5800020.0 }, + "xPixelSize": 30.0, + "yPixelSize": -30.0 + }, + "gridBounds": { + "min": [0, 0], + "max": [7999, 7999] + } + }, + "state": "source" + }, + "bands": [ + { "asset_title": "Blue (band 2) - 30m", "band_name": null }, + { "asset_title": "Green (band 3) - 30m", "band_name": null }, + { "asset_title": "NIR (band 5) - 30m", "band_name": null }, + { "asset_title": "Red (band 4) - 30m", "band_name": null } + ] + } + ] +} diff --git a/geoengine/test_data/stac_responses/items/code-de-harvest-test.json b/geoengine/test_data/stac_responses/items/code-de-harvest-test.json new file mode 100644 index 0000000000..d4160b9631 --- /dev/null +++ b/geoengine/test_data/stac_responses/items/code-de-harvest-test.json @@ -0,0 +1,200 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "id": "S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638", + "collection": "sentinel-2-l2a", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [8.7, 50.8], + [8.8, 50.8], + [8.8, 50.9], + [8.7, 50.9], + [8.7, 50.8] + ] + ] + }, + "bbox": [8.7, 50.8, 8.8, 50.9], + "properties": { + "datetime": "2026-07-24T15:36:29.024000Z", + "updated": "2026-07-24T19:29:11.371218Z" + }, + "assets": { + "B02_10m": { + "type": "image/jp2", + "title": "Blue (band 2) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B02_10m.jp2", + "bands": [{ "name": "B02", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B03_10m": { + "type": "image/jp2", + "title": "Green (band 3) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B03_10m.jp2", + "bands": [{ "name": "B03", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B04_10m": { + "type": "image/jp2", + "title": "Red (band 4) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B04_10m.jp2", + "bands": [{ "name": "B04", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B08_10m": { + "type": "image/jp2", + "title": "NIR 1 (band 8) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B08_10m.jp2", + "bands": [{ "name": "B08", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B11_20m": { + "type": "image/jp2", + "title": "SWIR 1 (band 11) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R20m/T19QBG_20260724T153629_B11_20m.jp2", + "bands": [{ "name": "B11", "eo:common_name": "swir16" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + }, + "B12_20m": { + "type": "image/jp2", + "title": "SWIR 2 (band 12) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R20m/T19QBG_20260724T153629_B12_20m.jp2", + "bands": [{ "name": "B12", "eo:common_name": "swir22" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + } + } + }, + { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "id": "S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638", + "collection": "sentinel-2-l2a", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [8.8, 50.8], + [8.9, 50.8], + [8.9, 50.9], + [8.8, 50.9], + [8.8, 50.8] + ] + ] + }, + "bbox": [8.8, 50.8, 8.9, 50.9], + "properties": { + "datetime": "2026-07-24T15:36:29.024000Z", + "updated": "2026-07-24T19:33:06.935907Z" + }, + "assets": { + "B02_10m": { + "type": "image/jp2", + "title": "Blue (band 2) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B02_10m.jp2", + "bands": [{ "name": "B02", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B03_10m": { + "type": "image/jp2", + "title": "Green (band 3) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B03_10m.jp2", + "bands": [{ "name": "B03", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B04_10m": { + "type": "image/jp2", + "title": "Red (band 4) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B04_10m.jp2", + "bands": [{ "name": "B04", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B08_10m": { + "type": "image/jp2", + "title": "NIR 1 (band 8) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B08_10m.jp2", + "bands": [{ "name": "B08", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B11_20m": { + "type": "image/jp2", + "title": "SWIR 1 (band 11) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R20m/T19QBF_20260724T153629_B11_20m.jp2", + "bands": [{ "name": "B11", "eo:common_name": "swir16" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + }, + "B12_20m": { + "type": "image/jp2", + "title": "SWIR 2 (band 12) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R20m/T19QBF_20260724T153629_B12_20m.jp2", + "bands": [{ "name": "B12", "eo:common_name": "swir22" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + } + } + } + ], + "links": [], + "numberReturned": 2, + "numberMatched": 2 +} diff --git a/geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json b/geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json new file mode 100644 index 0000000000..b22d4fcad8 --- /dev/null +++ b/geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json @@ -0,0 +1,82 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "id": "LC08_L1TP_192026_20260724_20260724_02_RT", + "collection": "landsat-c2-l1", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [8.5, 50.5], + [9.0, 50.5], + [9.0, 51.0], + [8.5, 51.0], + [8.5, 50.5] + ] + ] + }, + "bbox": [8.5, 50.5, 9.0, 51.0], + "properties": { + "datetime": "2026-07-24T10:32:18.024000Z", + "updated": "2026-07-24T18:15:42.137000Z" + }, + "assets": { + "B4_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red (band 4) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B4.TIF", + "bands": [{ "name": "B4", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + }, + "B5_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR (band 5) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B5.TIF", + "bands": [{ "name": "B5", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + }, + "B2_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Blue (band 2) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B2.TIF", + "bands": [{ "name": "B2", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + }, + "B3_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Green (band 3) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B3.TIF", + "bands": [{ "name": "B3", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + } + } + } + ], + "links": [], + "numberReturned": 1, + "numberMatched": 1 +} From 85cbed4ad8ef74b6ce0e5f5ebd704ba9b30ad3bf Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 28 Jul 2026 10:23:54 +0200 Subject: [PATCH 2/9] remove dynamic datasets --- .../src/cli/stac_harvester/harvest.rs | 135 +++--------------- 1 file changed, 23 insertions(+), 112 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 19fb812932..7c58e98de1 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -246,7 +246,6 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err ); let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut dynamic_datasets: HashMap = HashMap::new(); let mut items_processed: u64 = 0; let mut items_per_sec: f64; @@ -255,18 +254,12 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err let item_collection = result?; for item in &item_collection.items { - process_harvest_item_dynamic( - item, - provider_def, - &mut tiles_by_dataset, - &mut dynamic_datasets, - ¶ms, - ) - .unwrap_or_else(|e| { - if params.verbose { - warn!("Skipping item {}: {}", item.id, e); - } - }); + process_harvest_item(item, provider_def, &mut tiles_by_dataset, ¶ms) + .unwrap_or_else(|e| { + if params.verbose { + warn!("Skipping item {}: {}", item.id, e); + } + }); items_processed += 1; } @@ -308,35 +301,6 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err info!("Processed {} items total", items_processed); - // Create any dynamic (per-EPSG) datasets that were discovered during item processing - for (dyn_dataset_name, dyn_dataset) in &dynamic_datasets { - if !dataset_exists_api( - &client, - ¶ms.geo_engine_url, - &session_id, - dyn_dataset_name, - ) - .await? - { - if params.verbose { - info!("Creating dynamic dataset '{}'", dyn_dataset_name); - } - create_dataset_api( - &client, - ¶ms.geo_engine_url, - &session_id, - dyn_dataset_name, - dyn_dataset, - ¶ms.volume_name, - ) - .await?; - } - // Also ensure tiles_by_dataset has an entry for this dataset - tiles_by_dataset - .entry(dyn_dataset_name.clone()) - .or_default(); - } - for (dataset_name, tiles) in &tiles_by_dataset { if tiles.is_empty() { continue; @@ -395,16 +359,16 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err // Item Processing (Harvest) // --------------------------------------------------------------------------- -/// Like `process_harvest_item`, but handles items whose EPSG code differs from -/// the mapping's projection by dynamically creating per-EPSG dataset variants. -/// Items that pass the `--epsgs` filter but have a different EPSG than the -/// mapping will be grouped into separate datasets named with their actual EPSG. +/// Process a STAC item and add tiles to the appropriate dataset from the mapping. +/// +/// Only assets whose EPSG code matches the dataset's projection are included. +/// Items with a different EPSG are silently skipped — all datasets must be +/// predefined in the mapping. #[allow(clippy::too_many_lines)] -fn process_harvest_item_dynamic( +fn process_harvest_item( item: &stac::Item, provider_def: &StacDataProviderDefinition, tiles_by_dataset: &mut HashMap>, - dynamic_datasets: &mut HashMap, params: &StacHarvest, ) -> Result<(), anyhow::Error> { let Some(datetime) = item.properties.datetime else { @@ -465,43 +429,15 @@ fn process_harvest_item_dynamic( continue; }; - // Determine the actual projection and dataset name for this item - let (_actual_projection, actual_dataset_name) = if dataset.projection - == SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) + // Only process assets whose EPSG matches the dataset's projection + if dataset.projection + != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) { - // EPSG matches the mapping — use the dataset as-is - ( - dataset.projection, - dataset_name_for_harvest(&provider_def.collection_name, dataset), - ) - } else { - // EPSG differs — create a per-EPSG variant - let per_epsg_projection = - SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg); - - // Build a modified dataset with the item's EPSG - let per_epsg_dataset = StacProviderDataset { - projection: per_epsg_projection, - ..dataset.clone() - }; + continue; + } - let dyn_name = - dataset_name_for_harvest(&provider_def.collection_name, &per_epsg_dataset); - - // Register this dynamic dataset so it gets created - dynamic_datasets - .entry(dyn_name.clone()) - .or_insert_with(|| StacProviderDataset { - projection: SpatialReference::new( - SpatialReferenceAuthority::Epsg, - item_epsg, - ), - spatial_grid: dataset.spatial_grid, - ..dataset.clone() - }); - - (per_epsg_projection, dyn_name) - }; + let actual_dataset_name = + dataset_name_for_harvest(&provider_def.collection_name, dataset); let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) else { @@ -1326,19 +1262,12 @@ mod tests { }; let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut dynamic_datasets: HashMap = HashMap::new(); // Process the first item from the fixture let item = &items.items[0]; - process_harvest_item_dynamic( - item, - &mapping, - &mut tiles_by_dataset, - &mut dynamic_datasets, - ¶ms, - ) - .expect("item processing should succeed"); + process_harvest_item(item, &mapping, &mut tiles_by_dataset, ¶ms) + .expect("item processing should succeed"); // The mapping has 2 datasets (10m and 20m), the item has assets for both assert_eq!( @@ -1370,12 +1299,6 @@ mod tests { } } - // Verify no dynamic (per-EPSG) datasets since all items match the mapping EPSG - assert!( - dynamic_datasets.is_empty(), - "no dynamic datasets should be needed when EPSGs match" - ); - // 10m dataset should have 4 tiles (B02, B03, B04, B08) let total_tiles_10m: usize = tiles_by_dataset .iter() @@ -1430,18 +1353,11 @@ mod tests { }; let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut dynamic_datasets: HashMap = HashMap::new(); let item = &items.items[0]; - process_harvest_item_dynamic( - item, - &mapping, - &mut tiles_by_dataset, - &mut dynamic_datasets, - ¶ms, - ) - .expect("Landsat item processing should succeed"); + process_harvest_item(item, &mapping, &mut tiles_by_dataset, ¶ms) + .expect("Landsat item processing should succeed"); // Mapping has 1 dataset (30m), the item has assets for it assert_eq!( @@ -1472,11 +1388,6 @@ mod tests { } } - assert!( - dynamic_datasets.is_empty(), - "no dynamic datasets should be needed when EPSGs match" - ); - // 30m dataset should have 4 tiles (Blue, Green, Red, NIR) let total_tiles_30m: usize = tiles_by_dataset .iter() From d3ddde0614f0597add4d1dd72e16d6666e192763 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 28 Jul 2026 12:39:04 +0200 Subject: [PATCH 3/9] refactor --- .../src/cli/stac_harvester/discover.rs | 328 ++++++---- .../src/cli/stac_harvester/harvest.rs | 603 ++++++++++-------- 2 files changed, 527 insertions(+), 404 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index b2aec57207..5d24ee9fe0 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -103,7 +103,6 @@ pub enum ImportFileType { // Discover Mapping Implementation // --------------------------------------------------------------------------- -#[allow(clippy::too_many_lines)] pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), anyhow::Error> { let client = reqwest::Client::new(); @@ -122,11 +121,104 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), .await .context("Failed to fetch STAC collection")?; - // Scan collection-level item_assets for bands (partial information) + let dataset_bands = scan_collection_bands(&collection, ¶ms.file_types); + + if params.verbose { + info!( + "Found {} data type/resolution combinations from collection metadata", + dataset_bands.len() + ); + } + + let items_response = fetch_sample_items(&client, ¶ms).await?; + + if items_response.items.is_empty() { + anyhow::bail!("No items found in the collection. Cannot discover mapping."); + } + + info!( + "Probing {} sample item(s) to discover EPSG codes and additional bands", + items_response.items.len() + ); + + let (discovered_datasets, sample_band_info) = + process_sample_assets(&items_response, ¶ms.file_types, ¶ms.epsgs); + + if discovered_datasets.is_empty() { + anyhow::bail!( + "No matching assets found in sample items. Check your --file-types and --epsgs filters." + ); + } + + let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) + .map_err(|e| anyhow::anyhow!("{e}"))?; + + let s3_config = params + .s3_endpoint + .as_ref() + .map(|endpoint| StacProviderS3Config { + endpoint: endpoint.clone(), + access_key: params.s3_access_key.clone(), + secret_key: params.s3_secret_key.clone(), + }); + + let datasets = build_datasets( + &discovered_datasets, + &dataset_bands, + &sample_band_info, + params.full_projection_grid, + ¶ms.stac_collection, + ); + + let provider_def = StacDataProviderDefinition { + name: format!("{} from STAC", params.stac_collection), + id: DataProviderId::new(), + description: format!( + "Auto-discovered mapping for STAC collection '{}' at {}", + params.stac_collection, params.stac_url + ), + priority: Some(50), + api_url: params.stac_url.clone(), + collection_name: params.stac_collection.clone(), + s3_config, + time_dimension, + datasets, + }; + + let json = serde_json::to_string_pretty(&provider_def) + .context("Failed to serialize mapping to JSON")?; + + if let Some(output_path) = ¶ms.output { + std::fs::write(output_path, &json) + .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; + println!("Mapping written to {}", output_path.display()); + } else { + println!("{json}"); + } + + Ok(()) +} + +struct DiscoveredDatasetInfo { + geo_transform: Option, + proj_shape: Option<(usize, usize)>, + srs: SpatialReference, + asset_count: u32, +} + +// --------------------------------------------------------------------------- +// Discover Helper Functions +// --------------------------------------------------------------------------- + +/// Scan the collection-level `item_assets` for band metadata (data type, resolution, band names). +fn scan_collection_bands( + collection: &stac::Collection, + file_types: &[ImportFileType], +) -> HashMap> { let mut dataset_bands: HashMap> = HashMap::new(); for (_asset_key, asset) in &collection.item_assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { continue; } @@ -137,14 +229,14 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), } } - if params.verbose { - info!( - "Found {} data type/resolution combinations from collection metadata", - dataset_bands.len() - ); - } + dataset_bands +} - // Sample items to discover EPSG codes and additional band/resolution info +/// Build query parameters and fetch sample items from the STAC items API. +async fn fetch_sample_items( + client: &reqwest::Client, + params: &StacDiscoverMapping, +) -> Result { let items_url = format!( "{}/collections/{}/items", params.stac_url.trim_end_matches('/'), @@ -169,20 +261,21 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), query_params.push(("limit".to_string(), format!("{}", params.sample_items))); - let items_response: stac::ItemCollection = - stac_api_request_with_params(&client, &items_url, &query_params) - .await - .context("Failed to fetch sample items")?; - - if items_response.items.is_empty() { - anyhow::bail!("No items found in the collection. Cannot discover mapping."); - } + stac_api_request_with_params(client, &items_url, &query_params) + .await + .context("Failed to fetch sample items") +} - info!( - "Probing {} sample item(s) to discover EPSG codes and additional bands", - items_response.items.len() - ); +/// Process sample items to discover unique datasets (by EPSG, data type, resolution) +/// and their associated bands. +type DiscoveredDatasets = HashMap; +type SampleBandInfo = HashMap>; +fn process_sample_assets( + items_response: &stac::ItemCollection, + file_types: &[ImportFileType], + epsgs: &[u32], +) -> (DiscoveredDatasets, SampleBandInfo) { let mut discovered_datasets: HashMap = HashMap::new(); let mut sample_band_info: HashMap> = HashMap::new(); @@ -190,7 +283,7 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), let item_epsg = common::epsg_code_from_item(item, common::StacExtensionMajorVersion::V2); for (asset_key, asset) in &item.assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { continue; } @@ -214,7 +307,7 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), continue; }; - if !params.epsgs.is_empty() && !params.epsgs.contains(&epsg) { + if !epsgs.is_empty() && !epsgs.contains(&epsg) { continue; } @@ -254,28 +347,21 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), } } - if discovered_datasets.is_empty() { - anyhow::bail!( - "No matching assets found in sample items. Check your --file-types and --epsgs filters." - ); - } - - // Build the StacDataProviderDefinition - let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) - .map_err(|e| anyhow::anyhow!("{e}"))?; - - let s3_config = params - .s3_endpoint - .as_ref() - .map(|endpoint| StacProviderS3Config { - endpoint: endpoint.clone(), - access_key: params.s3_access_key.clone(), - secret_key: params.s3_secret_key.clone(), - }); + (discovered_datasets, sample_band_info) +} +/// Build the `StacProviderDataset` list from discovered datasets, collection bands, +/// and sample band information. +fn build_datasets( + discovered_datasets: &HashMap, + dataset_bands: &HashMap>, + sample_band_info: &HashMap>, + full_projection_grid: bool, + stac_collection: &str, +) -> Vec { let mut datasets: Vec = Vec::new(); - for (dataset_key, info) in &discovered_datasets { + for (dataset_key, info) in discovered_datasets { let partial_key = PartialDatasetKey { data_type: dataset_key.data_type, resolution: dataset_key.resolution, @@ -312,70 +398,16 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), bands.sort_by(|a, b| a.asset_title.cmp(&b.asset_title)); - let spatial_grid = if params.full_projection_grid { - // Compute grid bounds covering the full projected CRS extent - if let Some(gt) = info.geo_transform { - let grid_bounds = projection_grid_bounds(gt, dataset_key.epsg) - .unwrap_or_else(|| { - // Fallback: use first asset's shape - if let Some((height, width)) = info.proj_shape { - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .expect("fallback grid bounds should be valid") - } else { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") - } - }); - GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) - } else { - GeoOpSpatialGridDescriptor::source_from_parts( - GeoTransform::new( - (0.0, 0.0).into(), - dataset_key.resolution.into_inner(), - -dataset_key.resolution.into_inner(), - ), - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid"), - ) - } - } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { - GeoOpSpatialGridDescriptor::source_from_parts( - gt, - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .unwrap_or_else(|_| { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") - }), - ) - } else { - GeoOpSpatialGridDescriptor::source_from_parts( - GeoTransform::new( - (0.0, 0.0).into(), - dataset_key.resolution.into_inner(), - -dataset_key.resolution.into_inner(), - ), - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid"), - ) - }; + let spatial_grid = build_dataset_spatial_grid(info, dataset_key, full_projection_grid); let dataset_name = format!( "{} EPSG:{} {:?} {}m", - params.stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution + stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution ); datasets.push(StacProviderDataset { name: dataset_name, - description: format!( - "Auto-discovered from STAC collection '{}'", - params.stac_collection - ), + description: format!("Auto-discovered from STAC collection '{stac_collection}'"), data_type: dataset_key.data_type, resolution: SpatialResolution::new_unchecked( dataset_key.resolution.into_inner(), @@ -387,40 +419,65 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), }); } - let provider_def = StacDataProviderDefinition { - name: format!("{} from STAC", params.stac_collection), - id: DataProviderId::new(), - description: format!( - "Auto-discovered mapping for STAC collection '{}' at {}", - params.stac_collection, params.stac_url - ), - priority: Some(50), - api_url: params.stac_url.clone(), - collection_name: params.stac_collection.clone(), - s3_config, - time_dimension, - datasets, - }; + datasets +} - let json = serde_json::to_string_pretty(&provider_def) - .context("Failed to serialize mapping to JSON")?; +/// Build the spatial grid descriptor for a discovered dataset, optionally using the +/// full projected CRS extent instead of the first asset's shape. +fn build_dataset_spatial_grid( + info: &DiscoveredDatasetInfo, + dataset_key: &DatasetKey, + full_projection_grid: bool, +) -> GeoOpSpatialGridDescriptor { + let default_grid = || { + GeoOpSpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0.0, 0.0).into(), + dataset_key.resolution.into_inner(), + -dataset_key.resolution.into_inner(), + ), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid"), + ) + }; - if let Some(output_path) = ¶ms.output { - std::fs::write(output_path, &json) - .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; - println!("Mapping written to {}", output_path.display()); + if full_projection_grid { + if let Some(gt) = info.geo_transform { + let grid_bounds = projection_grid_bounds(gt, dataset_key.epsg) + .unwrap_or_else(|| fallback_grid_bounds(info)); + GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) + } else { + default_grid() + } + } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { + GeoOpSpatialGridDescriptor::source_from_parts( + gt, + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .unwrap_or_else(|_| { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + }), + ) } else { - println!("{json}"); + default_grid() } - - Ok(()) } -struct DiscoveredDatasetInfo { - geo_transform: Option, - proj_shape: Option<(usize, usize)>, - srs: SpatialReference, - asset_count: u32, +/// Fallback grid bounds: use the first asset's shape, or a single-pixel grid. +fn fallback_grid_bounds(info: &DiscoveredDatasetInfo) -> GridBoundingBox2D { + if let Some((height, width)) = info.proj_shape { + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .expect("fallback grid bounds should be valid") + } else { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + } } /// Compute grid bounds that cover the full projected CRS extent for the given @@ -451,7 +508,11 @@ fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option Result { // Harvest Implementation // --------------------------------------------------------------------------- -#[allow(clippy::too_many_lines)] pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Error> { let start_time = Instant::now(); @@ -164,28 +165,7 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err ) .await?; - let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); - - for (idx, dataset) in provider_def.datasets.iter().enumerate() { - let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); - - if params.verbose { - info!("Checking dataset '{}'", dataset_name); - } - - if !dataset_exists_api(&client, ¶ms.geo_engine_url, &session_id, &dataset_name).await? { - create_dataset_api( - &client, - ¶ms.geo_engine_url, - &session_id, - &dataset_name, - dataset, - ¶ms.volume_name, - ) - .await?; - created_datasets.push((idx, dataset.clone())); - } - } + let created_datasets = setup_datasets(&client, ¶ms, provider_def, &session_id).await?; info!( "Created {} new dataset(s) out of {}", @@ -199,145 +179,21 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err stac_api_url, provider_def.collection_name ); - let mut query_params: Vec<(String, String)> = Vec::new(); - - if let Some(bbox) = ¶ms.bbox - && bbox.len() == 4 - { - query_params.push(( - "bbox".to_string(), - format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), - )); - } - - if params.time_start.is_some() || params.time_end.is_some() { - query_params.push(( - "datetime".to_string(), - format!( - "{}/{}", - params.time_start.as_deref().unwrap_or(""), - params.time_end.as_deref().unwrap_or("") - ), - )); - } - - if let Some(limit) = params.limit { - query_params.push(("limit".to_string(), limit.to_string())); - } - - if params.filter_item_fields { - query_params.push(( - "fields".to_string(), - "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" - .to_string(), - )); - } - - let initial_query_state = QueryState::FirstPage { - query_url: items_url.clone(), - query_params, - }; - - let page_stream = create_page_stream( - initial_query_state, - client.clone(), - params.verbose, - params.prefetch_pages, - ); - - let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut items_processed: u64 = 0; - let mut items_per_sec: f64; - - futures::pin_mut!(page_stream); - while let Some(result) = page_stream.next().await { - let item_collection = result?; + let query_params = build_stac_query_params(¶ms); - for item in &item_collection.items { - process_harvest_item(item, provider_def, &mut tiles_by_dataset, ¶ms) - .unwrap_or_else(|e| { - if params.verbose { - warn!("Skipping item {}: {}", item.id, e); - } - }); - - items_processed += 1; - } - - if params.verbose { - let elapsed = start_time.elapsed().as_secs_f64(); - items_per_sec = if elapsed > 0.0 { - items_processed as f64 / elapsed - } else { - 0.0 - }; - - if let Some(number_matched) = item_collection - .additional_fields - .get("numberMatched") - .and_then(serde_json::Value::as_u64) - { - let progress = - (items_processed as f64 / number_matched as f64 * 100.0).clamp(0.0, 100.0); - let remaining = number_matched.saturating_sub(items_processed); - let eta_secs = if items_per_sec > 0.0 { - remaining as f64 / items_per_sec - } else { - f64::INFINITY - }; - let eta_str = if eta_secs.is_finite() { - format_duration(eta_secs as u64) - } else { - "unknown".to_string() - }; - println!( - "[{progress:.1}%] Processed {items_processed}/{number_matched} items ({items_per_sec:.1} items/s, ETA: {eta_str})" - ); - } else { - println!("Processed {items_processed} items ({items_per_sec:.1} items/s)"); - } - } - } + let (tiles_by_dataset, items_processed) = process_item_stream( + &client, + &items_url, + &query_params, + provider_def, + ¶ms, + &start_time, + ) + .await?; info!("Processed {} items total", items_processed); - for (dataset_name, tiles) in &tiles_by_dataset { - if tiles.is_empty() { - continue; - } - - if params.verbose { - info!("Adding {} tiles to dataset '{}'", tiles.len(), dataset_name); - } - - let batch_size = 100; - for chunk in tiles.chunks(batch_size) { - let response = retry_http( - || async { - client - .post(format!( - "{}/dataset/{}/tiles", - params.geo_engine_url, dataset_name - )) - .header("Content-Type", "application/json") - .header("Authorization", format!("Bearer {session_id}")) - .json(chunk) - .send() - .await - }, - &format!("Add tiles to dataset '{dataset_name}'"), - ) - .await - .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - warn!("Failed to add tiles to dataset '{dataset_name}' (HTTP {status}): {body}"); - // Continue with remaining tiles; some conflicts (e.g. z-index) are expected - } - } - } + upload_tiles_to_datasets(&client, ¶ms, &session_id, &tiles_by_dataset).await?; create_harvest_layer_collections( &client, @@ -364,7 +220,6 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err /// Only assets whose EPSG code matches the dataset's projection are included. /// Items with a different EPSG are silently skipped — all datasets must be /// predefined in the mapping. -#[allow(clippy::too_many_lines)] fn process_harvest_item( item: &stac::Item, provider_def: &StacDataProviderDefinition, @@ -396,119 +251,17 @@ fn process_harvest_item( for dataset in &provider_def.datasets { for (band_idx, band_def) in dataset.bands.iter().enumerate() { - let Some((_asset_key, asset)) = item - .assets - .iter() - .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title)) - else { - continue; - }; - - // Check data type matches - if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) - && asset_dt != dataset.data_type - { - continue; - } - - // Extract the item's actual EPSG code from the asset - let item_epsg = common::epsg_code_from_fields( - common::StacExtensionMajorVersion::V2, - &asset.additional_fields, - ) - .or_else(|| { - // Also try to extract from serialized properties as fallback - let props_val = serde_json::to_value(&item.properties) - .ok() - .and_then(|v| v.as_object().cloned()) - .unwrap_or_default(); - common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) - }); - - let Some(item_epsg) = item_epsg else { - continue; - }; - - // Only process assets whose EPSG matches the dataset's projection - if dataset.projection - != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) - { - continue; - } - - let actual_dataset_name = - dataset_name_for_harvest(&provider_def.collection_name, dataset); - - let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) - else { - continue; - }; - - if (geo_transform.x_pixel_size().abs() - dataset.resolution.x).abs() > 1e-9 - || (geo_transform.y_pixel_size().abs() - dataset.resolution.y).abs() > 1e-9 - { - continue; + if let Some((dataset_name, tile)) = try_create_tile_for_band( + item, + dataset, + band_idx, + band_def, + (time, z_index), + provider_def, + params, + ) { + tiles_by_dataset.entry(dataset_name).or_default().push(tile); } - - let Some((height, width)) = common::proj_shape_from_fields(&asset.additional_fields) - else { - continue; - }; - - let Some(rasterband_channel) = - common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref()) - else { - continue; - }; - - let grid_bounds = GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .context("Failed to create grid bounds")?; - - let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - - let Some(file_path) = common::gdal_file_path(&asset.href) else { - continue; - }; - - let gdal_config_options = common::gdal_config_options_for_file_path( - &file_path, - provider_def.s3_config.as_ref(), - ); - - let tile = AddDatasetTile { - time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) - .context("Failed to create time interval")? - .into(), - spatial_partition: spatial_partition.into(), - band: band_idx as u32, - z_index, - params: GdalDatasetParameters { - file_path, - rasterband_channel, - geo_transform: geo_transform.into(), - width, - height, - file_not_found_handling: - crate::api::model::operators::FileNotFoundHandling::Error, - no_data_value: params.no_data_value, - properties_mapping: None, - gdal_open_options: None, - gdal_config_options: gdal_config_options.map(|opts| { - opts.into_iter() - .map(|(k, v)| GdalConfigOption::from((k, v))) - .collect() - }), - allow_alphaband_as_mask: false, - }, - }; - - tiles_by_dataset - .entry(actual_dataset_name) - .or_default() - .push(tile); } } @@ -1190,6 +943,312 @@ fn data_type_from_asset_v1_1_0_fallback(asset: &stac::Asset) -> Option Result, anyhow::Error> { + let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); + + for (idx, dataset) in provider_def.datasets.iter().enumerate() { + let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + + if params.verbose { + info!("Checking dataset '{}'", dataset_name); + } + + if !dataset_exists_api(client, ¶ms.geo_engine_url, session_id, &dataset_name).await? { + create_dataset_api( + client, + ¶ms.geo_engine_url, + session_id, + &dataset_name, + dataset, + ¶ms.volume_name, + ) + .await?; + created_datasets.push((idx, dataset.clone())); + } + } + + Ok(created_datasets) +} + +/// Build the query parameters for the STAC items API request. +fn build_stac_query_params(params: &StacHarvest) -> Vec<(String, String)> { + let mut query_params: Vec<(String, String)> = Vec::new(); + + if let Some(bbox) = ¶ms.bbox + && bbox.len() == 4 + { + query_params.push(( + "bbox".to_string(), + format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), + )); + } + + if params.time_start.is_some() || params.time_end.is_some() { + query_params.push(( + "datetime".to_string(), + format!( + "{}/{}", + params.time_start.as_deref().unwrap_or(""), + params.time_end.as_deref().unwrap_or("") + ), + )); + } + + if let Some(limit) = params.limit { + query_params.push(("limit".to_string(), limit.to_string())); + } + + if params.filter_item_fields { + query_params.push(( + "fields".to_string(), + "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" + .to_string(), + )); + } + + query_params +} + +/// Fetch pages of STAC items, process them, and collect tiles grouped by dataset. +async fn process_item_stream( + client: &reqwest::Client, + items_url: &str, + query_params: &[(String, String)], + provider_def: &StacDataProviderDefinition, + params: &StacHarvest, + start_time: &Instant, +) -> Result<(HashMap>, u64), anyhow::Error> { + let initial_query_state = QueryState::FirstPage { + query_url: items_url.to_string(), + query_params: query_params.to_vec(), + }; + + let page_stream = create_page_stream( + initial_query_state, + client.clone(), + params.verbose, + params.prefetch_pages, + ); + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut items_processed: u64 = 0; + let mut items_per_sec: f64; + + futures::pin_mut!(page_stream); + while let Some(result) = page_stream.next().await { + let item_collection = result?; + + for item in &item_collection.items { + process_harvest_item(item, provider_def, &mut tiles_by_dataset, params).unwrap_or_else( + |e| { + if params.verbose { + warn!("Skipping item {}: {}", item.id, e); + } + }, + ); + + items_processed += 1; + } + + if params.verbose { + let elapsed = start_time.elapsed().as_secs_f64(); + items_per_sec = if elapsed > 0.0 { + items_processed as f64 / elapsed + } else { + 0.0 + }; + + if let Some(number_matched) = item_collection + .additional_fields + .get("numberMatched") + .and_then(serde_json::Value::as_u64) + { + let progress = + (items_processed as f64 / number_matched as f64 * 100.0).clamp(0.0, 100.0); + let remaining = number_matched.saturating_sub(items_processed); + let eta_secs = if items_per_sec > 0.0 { + remaining as f64 / items_per_sec + } else { + f64::INFINITY + }; + let eta_str = if eta_secs.is_finite() { + format_duration(eta_secs as u64) + } else { + "unknown".to_string() + }; + println!( + "[{progress:.1}%] Processed {items_processed}/{number_matched} items ({items_per_sec:.1} items/s, ETA: {eta_str})" + ); + } else { + println!("Processed {items_processed} items ({items_per_sec:.1} items/s)"); + } + } + } + + Ok((tiles_by_dataset, items_processed)) +} + +/// Upload collected tiles to the Geo Engine server in batches. +async fn upload_tiles_to_datasets( + client: &reqwest::Client, + params: &StacHarvest, + session_id: &str, + tiles_by_dataset: &HashMap>, +) -> Result<(), anyhow::Error> { + for (dataset_name, tiles) in tiles_by_dataset { + if tiles.is_empty() { + continue; + } + + if params.verbose { + info!("Adding {} tiles to dataset '{}'", tiles.len(), dataset_name); + } + + let batch_size = 100; + for chunk in tiles.chunks(batch_size) { + let response = retry_http( + || async { + client + .post(format!( + "{}/dataset/{}/tiles", + params.geo_engine_url, dataset_name + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(chunk) + .send() + .await + }, + &format!("Add tiles to dataset '{dataset_name}'"), + ) + .await + .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + warn!("Failed to add tiles to dataset '{dataset_name}' (HTTP {status}): {body}"); + // Continue with remaining tiles; some conflicts (e.g. z-index) are expected + } + } + } + + Ok(()) +} + +/// Try to create a tile for a single band of a single dataset from a STAC item asset. +/// +/// Returns `None` if no matching asset exists, the data types don't match, the EPSG code +/// doesn't match the dataset's projection, the resolution doesn't match, or any required +/// metadata is missing. +fn try_create_tile_for_band( + item: &stac::Item, + dataset: &StacProviderDataset, + band_idx: usize, + band_def: &StacProviderDatasetBand, + item_time: (TimeInstance, i64), // (time, z_index) + provider_def: &StacDataProviderDefinition, + params: &StacHarvest, +) -> Option<(String, AddDatasetTile)> { + let (time, z_index) = item_time; + let (_asset_key, asset) = item + .assets + .iter() + .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title))?; + + // Check data type matches + if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) + && asset_dt != dataset.data_type + { + return None; + } + + // Extract the item's actual EPSG code from the asset + let item_epsg = common::epsg_code_from_fields( + common::StacExtensionMajorVersion::V2, + &asset.additional_fields, + ) + .or_else(|| { + // Also try to extract from serialized properties as fallback + let props_val = serde_json::to_value(&item.properties) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(); + common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) + })?; + + // Only process assets whose EPSG matches the dataset's projection + if dataset.projection != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) { + return None; + } + + let actual_dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + + let geo_transform = common::geo_transform_from_fields(&asset.additional_fields)?; + + if (geo_transform.x_pixel_size().abs() - dataset.resolution.x).abs() > 1e-9 + || (geo_transform.y_pixel_size().abs() - dataset.resolution.y).abs() > 1e-9 + { + return None; + } + + let (height, width) = common::proj_shape_from_fields(&asset.additional_fields)?; + + let rasterband_channel = + common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref())?; + + let grid_bounds = GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .ok()?; + + let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); + + let file_path = common::gdal_file_path(&asset.href)?; + + let gdal_config_options = + common::gdal_config_options_for_file_path(&file_path, provider_def.s3_config.as_ref()); + + let tile = AddDatasetTile { + time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) + .ok()? + .into(), + spatial_partition: spatial_partition.into(), + band: band_idx as u32, + z_index, + params: GdalDatasetParameters { + file_path, + rasterband_channel, + geo_transform: geo_transform.into(), + width, + height, + file_not_found_handling: crate::api::model::operators::FileNotFoundHandling::Error, + no_data_value: params.no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: gdal_config_options.map(|opts| { + opts.into_iter() + .map(|(k, v)| GdalConfigOption::from((k, v))) + .collect() + }), + allow_alphaband_as_mask: false, + }, + }; + + Some((actual_dataset_name, tile)) +} + fn format_duration(secs: u64) -> String { if secs < 60 { format!("{secs}s") From c2cf5762334aa816b363c55ad88ba5b321d3d527 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 28 Jul 2026 12:55:10 +0200 Subject: [PATCH 4/9] refactor --- .../src/cli/stac_harvester/discover.rs | 76 +++++++++---------- .../src/cli/stac_harvester/harvest.rs | 15 +--- .../src/datasets/external/stac/common.rs | 30 ++++++++ 3 files changed, 70 insertions(+), 51 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index 5d24ee9fe0..cb60d1d037 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -223,7 +223,7 @@ fn scan_collection_bands( } if let Ok(Some(bands)) = - scan_item_asset_common(&collection.version, asset, collection.summaries.as_ref()) + scan_collection_item_asset(&collection.version, asset, collection.summaries.as_ref()) { merge_dataset_bands(&mut dataset_bands, bands); } @@ -293,7 +293,7 @@ fn process_sample_assets( }; let data_type = common::data_type_from_asset_v1_1_0(asset) - .or_else(|| data_type_from_asset_v1_0_0_fallback(asset)); + .or_else(|| common::data_type_from_asset_v1_0_0_fallback(asset)); let Some(data_type) = data_type else { continue; }; @@ -436,8 +436,7 @@ fn build_dataset_spatial_grid( dataset_key.resolution.into_inner(), -dataset_key.resolution.into_inner(), ), - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid"), + zero_size_grid(), ) }; @@ -450,17 +449,8 @@ fn build_dataset_spatial_grid( default_grid() } } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { - GeoOpSpatialGridDescriptor::source_from_parts( - gt, - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .unwrap_or_else(|_| { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") - }), - ) + let grid_bounds = asset_shape_bounds(height, width).unwrap_or_else(|()| zero_size_grid()); + GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) } else { default_grid() } @@ -469,17 +459,31 @@ fn build_dataset_spatial_grid( /// Fallback grid bounds: use the first asset's shape, or a single-pixel grid. fn fallback_grid_bounds(info: &DiscoveredDatasetInfo) -> GridBoundingBox2D { if let Some((height, width)) = info.proj_shape { - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .expect("fallback grid bounds should be valid") + asset_shape_bounds(height, width).unwrap_or_else(|()| zero_size_grid()) } else { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") + zero_size_grid() } } +/// Build grid bounds from an asset's `(height, width)` shape. +/// Returns `Err` if dimensions are zero (causing negative indices). +fn asset_shape_bounds(height: usize, width: usize) -> Result { + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([ + width.saturating_sub(1) as isize, + height.saturating_sub(1) as isize, + ]), + ) + .map_err(|_| ()) +} + +/// A single-pixel grid at the origin, used as a safe fallback when no shape info is available. +fn zero_size_grid() -> GridBoundingBox2D { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should always be valid") +} + /// Compute grid bounds that cover the full projected CRS extent for the given /// geo-transform and EPSG code. Currently handles UTM projections (zones 32601–32660 /// and 32701–32760) with known extents. Returns `None` for unsupported CRS types. @@ -548,19 +552,24 @@ struct PartialDatasetKey { // Collection scanning helpers // --------------------------------------------------------------------------- -fn scan_item_asset_common( +fn scan_collection_item_asset( collection_version: &stac::Version, asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, ) -> Result>>, String> { match collection_version { - stac::Version::v1_0_0 => scan_item_asset_v1_0_0_common(asset), - stac::Version::v1_1_0 => scan_item_asset_v1_1_0_common(asset, collection_summaries), - _ => Err(format!("Unsupported STAC version: {collection_version}")), + stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset), + stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries), + _ => { + // For unknown STAC versions, try v1.1.0 first (more common), fall back to v1.0.0 + scan_collection_item_asset_v1_1_0(asset, collection_summaries) + .or_else(|_| scan_collection_item_asset_v1_0_0(asset)) + .or(Ok(None)) + } } } -fn scan_item_asset_v1_0_0_common( +fn scan_collection_item_asset_v1_0_0( asset: &stac::ItemAsset, ) -> Result>>, String> { let mut dataset_bands: HashMap> = HashMap::new(); @@ -625,7 +634,7 @@ fn scan_item_asset_v1_0_0_common( Ok(Some(dataset_bands)) } -fn scan_item_asset_v1_1_0_common( +fn scan_collection_item_asset_v1_1_0( asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, ) -> Result>>, String> { @@ -678,17 +687,6 @@ fn scan_item_asset_v1_1_0_common( Ok(Some(dataset_bands)) } -fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option { - asset - .additional_fields - .get("raster:bands") - .and_then(|v| v.as_array()) - .and_then(|bands| bands.first()) - .and_then(|band| band.get("data_type")) - .and_then(|v| v.as_str()) - .and_then(common::raster_data_type_from_stac_data_type_str) -} - fn matches_selected_file_types_static( media_type: Option<&str>, file_types: &[ImportFileType], diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 112af0ad81..0c1767e578 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -11,7 +11,7 @@ use futures::StreamExt; use geoengine_datatypes::{ dataset::NamedData, primitives::{DateTime, TimeInstance, TimeInterval}, - raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, + raster::{GeoTransform, GridBoundingBox2D, GridIdx2D}, spatial_reference::{SpatialReference, SpatialReferenceAuthority, SpatialReferenceOption}, }; use tracing::{debug, error, info, warn}; @@ -933,16 +933,6 @@ async fn login_geo_engine( Ok((client, session_id)) } -fn data_type_from_asset_v1_1_0_fallback(asset: &stac::Asset) -> Option { - common::data_type_from_asset_v1_1_0(asset).or_else(|| { - asset - .additional_fields - .get("data_type") - .and_then(|v| v.as_str()) - .and_then(common::raster_data_type_from_stac_data_type_str) - }) -} - // --------------------------------------------------------------------------- // Harvest Helper Functions // --------------------------------------------------------------------------- @@ -1167,7 +1157,7 @@ fn try_create_tile_for_band( .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title))?; // Check data type matches - if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) + if let Some(asset_dt) = common::data_type_from_asset_v1_1_0_fallback(asset) && asset_dt != dataset.data_type { return None; @@ -1267,6 +1257,7 @@ fn format_duration(secs: u64) -> String { mod tests { use super::*; use geoengine_datatypes::primitives::SpatialResolution; + use geoengine_datatypes::raster::RasterDataType; #[test] fn test_dataset_name_for_harvest() { diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index 89a47f905a..ed175ef630 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -219,6 +219,36 @@ pub fn data_type_from_asset_v1_1_0(asset: &stac::Asset) -> Option Option { + data_type_from_asset_v1_1_0(asset).or_else(|| { + asset + .additional_fields + .get("data_type") + .and_then(|v| v.as_str()) + .and_then(raster_data_type_from_stac_data_type_str) + }) +} + +/// Extract data type from a STAC 1.0.0 asset, reading from `additional_fields["raster:bands"][0]["data_type"]`. +/// +/// STAC 1.0.0 stores data types inside `raster:bands[]` arrays rather than directly on the asset. +/// This function reads the first band's `data_type` as a raw string. +pub fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option { + asset + .additional_fields + .get("raster:bands") + .and_then(|v| v.as_array()) + .and_then(|bands| bands.first()) + .and_then(|band| band.get("data_type")) + .and_then(|v| v.as_str()) + .and_then(raster_data_type_from_stac_data_type_str) +} + // --------------------------------------------------------------------------- // File path helpers // --------------------------------------------------------------------------- From c9f8a06ddc8a929ce5b356f13ee3749e755b0ca4 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Wed, 29 Jul 2026 16:07:08 +0200 Subject: [PATCH 5/9] add page limit --- geoengine/services/src/api/model/services.rs | 3 + .../src/cli/stac_harvester/discover.rs | 94 +++++-- .../src/cli/stac_harvester/harvest.rs | 235 +++++++++--------- .../contexts/migrations/current_schema.sql | 1 + .../migration_0028_stac_provider.sql | 1 + .../src/datasets/external/stac/listing.rs | 2 + .../datasets/external/stac/loading_info.rs | 13 +- .../src/datasets/external/stac/mod.rs | 6 +- .../provider_defs_api/stac_sentinel2.json | 3 +- .../expected-mapping-code-de.json | 43 ++-- .../expected-mapping-landsat-c2-l1.json | 29 +-- 11 files changed, 244 insertions(+), 186 deletions(-) diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 37e63c4b9d..922fa36342 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -1122,6 +1122,7 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, pub datasets: Vec, + pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, @@ -1145,6 +1146,7 @@ impl From s3_config: value.s3_config.map(Into::into), time_dimension: api_time_dimension_to_datatypes(value.time_dimension), datasets: value.datasets.into_iter().map(Into::into).collect(), + page_limit: value.page_limit, query_timeout_secs: value.query_timeout_secs, } } @@ -1165,6 +1167,7 @@ impl From s3_config: value.s3_config.map(Into::into), time_dimension: datatypes_time_dimension_to_api(value.time_dimension), datasets: value.datasets.into_iter().map(Into::into).collect(), + page_limit: value.page_limit, query_timeout_secs: value.query_timeout_secs, } } diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index cb60d1d037..dc35b68e0e 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -54,6 +54,10 @@ pub struct StacDiscoverMapping { #[arg(long, default_value_t = 5)] pub sample_items: usize, + /// Page size for querying items from the STAC server (default: 100) + #[arg(long, default_value_t = 100)] + pub page_limit: usize, + /// Output file for the mapping JSON (default: stdout) #[arg(long)] pub output: Option, @@ -183,10 +187,13 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), s3_config, time_dimension, datasets, + page_limit: params.page_limit as i64, }; - let json = serde_json::to_string_pretty(&provider_def) - .context("Failed to serialize mapping to JSON")?; + let json = serde_json::to_string_pretty( + &crate::api::model::services::StacDataProviderDefinition::from(provider_def), + ) + .context("Failed to serialize mapping to JSON")?; if let Some(output_path) = ¶ms.output { std::fs::write(output_path, &json) @@ -218,7 +225,7 @@ fn scan_collection_bands( let mut dataset_bands: HashMap> = HashMap::new(); for (_asset_key, asset) in &collection.item_assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { + if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { continue; } @@ -233,11 +240,13 @@ fn scan_collection_bands( } /// Build query parameters and fetch sample items from the STAC items API. +/// Follows pagination links until `sample_items` items are collected or no +/// more pages are available. async fn fetch_sample_items( client: &reqwest::Client, params: &StacDiscoverMapping, ) -> Result { - let items_url = format!( + let base_url = format!( "{}/collections/{}/items", params.stac_url.trim_end_matches('/'), params.stac_collection @@ -259,11 +268,42 @@ async fn fetch_sample_items( )); } - query_params.push(("limit".to_string(), format!("{}", params.sample_items))); + query_params.push(("limit".to_string(), params.page_limit.to_string())); - stac_api_request_with_params(client, &items_url, &query_params) - .await - .context("Failed to fetch sample items") + let mut all_items = Vec::new(); + let mut next_url: Option = None; + + loop { + let page = if let Some(ref url) = next_url { + stac_api_request_parse::(client, url) + .await + .context("Failed to fetch sample items page")? + } else { + stac_api_request_with_params(client, &base_url, &query_params) + .await + .context("Failed to fetch sample items")? + }; + + all_items.extend(page.items); + + if all_items.len() >= params.sample_items { + all_items.truncate(params.sample_items); + break; + } + + // Follow the `next` link if available + next_url = page + .links + .iter() + .find(|link| link.rel == "next") + .map(|link| link.href.clone()); + + if next_url.is_none() { + break; + } + } + + Ok(all_items.into()) } /// Process sample items to discover unique datasets (by EPSG, data type, resolution) @@ -283,7 +323,7 @@ fn process_sample_assets( let item_epsg = common::epsg_code_from_item(item, common::StacExtensionMajorVersion::V2); for (asset_key, asset) in &item.assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { + if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { continue; } @@ -519,18 +559,26 @@ fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option Option<(f64, f64, f64, f64)> { - // UTM northern hemisphere zones EPSG:32601 – 32660 - if (32601..=32660).contains(&epsg) { - return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); - } - // UTM southern hemisphere zones EPSG:32701 – 32760 - if (32701..=32760).contains(&epsg) { - return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); - } - None + use proj::Proj; + + let proj_crs = Proj::new(&format!("EPSG:{epsg}")).ok()?; + + // Get area of use in degrees (WGS84) + let (area, _name) = proj_crs.area_of_use().ok()?; + let area = area?; + + // Project the WGS84 bounding box into the target CRS + let pipeline = Proj::new_known_crs("EPSG:4326", &format!("EPSG:{epsg}"), None).ok()?; + let bounds = pipeline + .transform_bounds(area.west, area.south, area.east, area.north, 21) + .ok()?; + + // transform_bounds returns [west, south, east, north] in the target CRS + Some((bounds[0], bounds[2], bounds[1], bounds[3])) } /// A key that uniquely identifies a Geo Engine dataset derived from STAC assets. @@ -687,7 +735,7 @@ fn scan_collection_item_asset_v1_1_0( Ok(Some(dataset_bands)) } -fn matches_selected_file_types_static( +fn matches_selected_file_types( media_type: Option<&str>, file_types: &[ImportFileType], ) -> bool { @@ -850,6 +898,7 @@ mod tests { verbose: false, time_granularity: "days".to_string(), time_step: 1, + page_limit: 100, }; discover_mapping(params) @@ -863,7 +912,7 @@ mod tests { let mut output_json: serde_json::Value = serde_json::from_str(&output_content).expect("output should be valid JSON"); - // Normalize dynamic fields before comparison + // Normalize variable fields before comparison let output_obj = output_json.as_object_mut().unwrap(); output_obj.remove("id"); output_obj.insert( @@ -960,6 +1009,7 @@ mod tests { verbose: false, time_granularity: "days".to_string(), time_step: 1, + page_limit: 100, }; discover_mapping(params) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 0c1767e578..9b57c259bf 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -40,7 +40,7 @@ use crate::{ RasterBandDescriptors, RasterResultDescriptor, RegularTimeDimension, SpatialGridDescriptor, SpatialGridDescriptorState, TimeDescriptor, TimeDimension, }, - responses::{ErrorResponse, IdResponse}, + responses::IdResponse, services::{ AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, }, @@ -59,6 +59,7 @@ use geoengine_operators::{ engine::{RasterOperator, TypedOperator}, source::{MultiBandGdalSource, MultiBandGdalSourceParameters}, }; +use geoengine_api_client::apis::configuration::Configuration as ApiConfig; // --------------------------------------------------------------------------- // Harvest @@ -83,10 +84,6 @@ pub struct StacHarvest { #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ')] pub bbox: Option>, - /// Import limit (page size) - #[arg(long)] - pub limit: Option, - /// Geo Engine API URL #[arg(long, default_value = "http://localhost:3030/api")] pub geo_engine_url: String, @@ -139,7 +136,9 @@ fn parse_mapping_file(s: &str) -> Result { } else { std::fs::read_to_string(s).map_err(|e| format!("Failed to read mapping from '{s}': {e}"))? }; - serde_json::from_str(&json).map_err(|e| format!("Invalid mapping JSON: {e}")) + let api_def: crate::api::model::services::StacDataProviderDefinition = + serde_json::from_str(&json).map_err(|e| format!("Invalid mapping JSON: {e}"))?; + Ok(api_def.into()) } // --------------------------------------------------------------------------- @@ -158,14 +157,17 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err provider_def.datasets.len() ); - let (client, session_id) = login_geo_engine( + let api_config = create_api_config( ¶ms.geo_engine_url, ¶ms.geo_engine_email, ¶ms.geo_engine_password, ) .await?; - let created_datasets = setup_datasets(&client, ¶ms, provider_def, &session_id).await?; + // Separate reqwest client for STAC API calls (not Geo Engine) + let stac_client = reqwest::Client::new(); + + let created_datasets = setup_datasets(&api_config, ¶ms, provider_def).await?; info!( "Created {} new dataset(s) out of {}", @@ -179,10 +181,10 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err stac_api_url, provider_def.collection_name ); - let query_params = build_stac_query_params(¶ms); + let query_params = build_stac_query_params(¶ms, provider_def.page_limit as usize); let (tiles_by_dataset, items_processed) = process_item_stream( - &client, + &stac_client, &items_url, &query_params, provider_def, @@ -193,12 +195,10 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err info!("Processed {} items total", items_processed); - upload_tiles_to_datasets(&client, ¶ms, &session_id, &tiles_by_dataset).await?; + upload_tiles_to_datasets(&api_config, ¶ms, &tiles_by_dataset).await?; create_harvest_layer_collections( - &client, - ¶ms.geo_engine_url, - &session_id, + &api_config, provider_def, &created_datasets, ¶ms, @@ -306,47 +306,36 @@ fn dataset_name_for_harvest(collection_name: &str, dataset: &StacProviderDataset } async fn dataset_exists_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, dataset_name: &str, ) -> Result { - let response = retry_http( - || async { - client - .get(format!("{geo_engine_url}/dataset/{dataset_name}")) - .header("Authorization", format!("Bearer {session_id}")) - .send() - .await - }, + let result = retry_http( + || geoengine_api_client::apis::datasets_api::get_dataset_handler(api_config, dataset_name), &format!("Check dataset existence for '{dataset_name}'"), ) - .await?; - - if response.status().is_success() { - return Ok(true); - } - - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - if status == reqwest::StatusCode::BAD_REQUEST - && let Ok(error_response) = serde_json::from_str::(&body) - && error_response.error == "CannotLoadDataset" - { - return Ok(false); + .await; + + match result { + Ok(_) => Ok(true), + Err(geoengine_api_client::apis::Error::ResponseError(resp)) + if resp.status == reqwest::StatusCode::BAD_REQUEST => + { + Ok(false) + } + Err(e) => Err(anyhow::anyhow!("Failed to check dataset '{dataset_name}': {e}")), } - - anyhow::bail!("Failed to check dataset '{dataset_name}': HTTP {status}: {body}"); } async fn create_dataset_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, dataset_name: &str, dataset: &StacProviderDataset, volume_name: &str, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let bands: Vec = dataset .bands .iter() @@ -358,7 +347,6 @@ async fn create_dataset_api( }) .collect(); - // Get GeoTransform from the spatial grid descriptor for the API let dt_gt: GeoTransform = dataset.spatial_grid.geo_transform(); let api_gt: crate::api::model::datatypes::GeoTransform = dt_gt.into(); @@ -438,17 +426,17 @@ async fn create_dataset_api( dataset_name.to_string() }; - share_dataset_api(client, geo_engine_url, session_id, &created_name).await?; + share_dataset_api(api_config, &created_name).await?; Ok(()) } async fn share_dataset_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, dataset_name: &str, ) -> Result<(), anyhow::Error> { + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let permissions = vec![ PermissionRequest { resource: Resource::Dataset(DatasetResource { @@ -471,8 +459,9 @@ async fn share_dataset_api( for permission in &permissions { retry_http( || async { - client - .put(format!("{geo_engine_url}/permissions")) + api_config + .client + .put(format!("{}/permissions", api_config.base_path)) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {session_id}")) .json(permission) @@ -488,17 +477,17 @@ async fn share_dataset_api( } async fn create_harvest_layer_collections( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, provider_def: &StacDataProviderDefinition, created_datasets: &[(usize, StacProviderDataset)], params: &StacHarvest, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let root_collection_id = create_layer_collection_api( - client, - geo_engine_url, - session_id, + api_config, &LayerCollectionId(INTERNAL_LAYER_DB_ROOT_COLLECTION_ID.to_string()), &provider_def.collection_name, &format!( @@ -510,9 +499,7 @@ async fn create_harvest_layer_collections( .await?; let temp_collection_id = create_layer_collection_api( - client, - geo_engine_url, - session_id, + api_config, &root_collection_id, "_layers", "All dataset layers (internal)", @@ -567,23 +554,25 @@ async fn create_harvest_layer_collections( ) .await?; - share_layer_api(client, geo_engine_url, session_id, &response.id).await?; + share_layer_api(api_config, &response.id).await?; } Ok(()) } async fn create_layer_collection_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, parent_id: &LayerCollectionId, name: &str, description: &str, params: &StacHarvest, ) -> Result { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + if let Some(existing_id) = - find_child_collection_by_name(client, geo_engine_url, session_id, parent_id, name).await? + find_child_collection_by_name(api_config, parent_id, name).await? { if params.verbose { info!("Found existing layer collection '{name}'"); @@ -615,18 +604,20 @@ async fn create_layer_collection_api( ) .await?; - share_layer_collection_api(client, geo_engine_url, session_id, &response.id).await?; + share_layer_collection_api(api_config, &response.id).await?; Ok(response.id) } async fn find_child_collection_by_name( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, parent_id: &LayerCollectionId, child_name: &str, ) -> Result, anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let mut offset: u32 = 0; let limit: u32 = 20; @@ -665,11 +656,13 @@ async fn find_child_collection_by_name( } async fn share_layer_collection_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, collection_id: &LayerCollectionId, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let permissions = vec![ PermissionRequest { resource: Resource::LayerCollection(LayerCollectionResource { @@ -709,11 +702,13 @@ async fn share_layer_collection_api( } async fn share_layer_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, layer_id: &LayerId, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let permissions = vec![ PermissionRequest { resource: Resource::Layer(LayerResource { @@ -896,41 +891,34 @@ where // Authentication helper // --------------------------------------------------------------------------- -async fn login_geo_engine( +async fn create_api_config( geo_engine_url: &str, geo_engine_email: &str, geo_engine_password: &str, -) -> Result<(reqwest::Client, String), anyhow::Error> { - let client = reqwest::Client::new(); +) -> Result { + use geoengine_api_client::models; - let response = retry_http( - || async { - client - .post(format!("{geo_engine_url}/login")) - .header("Content-Type", "application/json") - .json(&serde_json::json!({ - "email": geo_engine_email, - "password": geo_engine_password, - })) - .send() - .await - }, + let config = ApiConfig { + base_path: geo_engine_url.to_string(), + ..ApiConfig::default() + }; + + let credentials = models::UserCredentials::new( + geo_engine_email.to_string(), + geo_engine_password.to_string(), + ); + + let session = retry_http( + || geoengine_api_client::apis::session_api::login_handler(&config, credentials.clone()), "Login to Geo Engine", ) .await .context("Failed to authenticate")?; - let json = response - .json::() - .await - .context("Failed to parse auth response")?; - - let session_id = json["id"] - .as_str() - .context("No session id in response")? - .to_string(); - - Ok((client, session_id)) + Ok(ApiConfig { + bearer_access_token: Some(session.id.to_string()), + ..config + }) } // --------------------------------------------------------------------------- @@ -939,10 +927,9 @@ async fn login_geo_engine( /// Create datasets that don't already exist on the Geo Engine server. async fn setup_datasets( - client: &reqwest::Client, + api_config: &ApiConfig, params: &StacHarvest, provider_def: &StacDataProviderDefinition, - session_id: &str, ) -> Result, anyhow::Error> { let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); @@ -953,11 +940,9 @@ async fn setup_datasets( info!("Checking dataset '{}'", dataset_name); } - if !dataset_exists_api(client, ¶ms.geo_engine_url, session_id, &dataset_name).await? { + if !dataset_exists_api(api_config, &dataset_name).await? { create_dataset_api( - client, - ¶ms.geo_engine_url, - session_id, + api_config, &dataset_name, dataset, ¶ms.volume_name, @@ -971,7 +956,7 @@ async fn setup_datasets( } /// Build the query parameters for the STAC items API request. -fn build_stac_query_params(params: &StacHarvest) -> Vec<(String, String)> { +fn build_stac_query_params(params: &StacHarvest, page_limit: usize) -> Vec<(String, String)> { let mut query_params: Vec<(String, String)> = Vec::new(); if let Some(bbox) = ¶ms.bbox @@ -994,9 +979,7 @@ fn build_stac_query_params(params: &StacHarvest) -> Vec<(String, String)> { )); } - if let Some(limit) = params.limit { - query_params.push(("limit".to_string(), limit.to_string())); - } + query_params.push(("limit".to_string(), page_limit.to_string())); if params.filter_item_fields { query_params.push(( @@ -1090,11 +1073,14 @@ async fn process_item_stream( /// Upload collected tiles to the Geo Engine server in batches. async fn upload_tiles_to_datasets( - client: &reqwest::Client, + api_config: &ApiConfig, params: &StacHarvest, - session_id: &str, tiles_by_dataset: &HashMap>, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + for (dataset_name, tiles) in tiles_by_dataset { if tiles.is_empty() { continue; @@ -1110,8 +1096,7 @@ async fn upload_tiles_to_datasets( || async { client .post(format!( - "{}/dataset/{}/tiles", - params.geo_engine_url, dataset_name + "{geo_engine_url}/dataset/{dataset_name}/tiles", )) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {session_id}")) @@ -1283,10 +1268,12 @@ mod tests { #[test] fn test_process_harvest_item_produces_correct_tiles() { - let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( - "../../../../test_data/stac_responses/expected-mapping-code-de.json" - )) - .expect("valid mapping fixture"); + let api_mapping: crate::api::model::services::StacDataProviderDefinition = + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid mapping fixture"); + let mapping: StacDataProviderDefinition = api_mapping.into(); let items: stac::ItemCollection = serde_json::from_str(include_str!( "../../../../test_data/stac_responses/items/code-de-harvest-test.json" @@ -1298,7 +1285,6 @@ mod tests { time_start: None, time_end: None, bbox: None, - limit: None, geo_engine_url: String::new(), geo_engine_email: String::new(), geo_engine_password: String::new(), @@ -1374,10 +1360,12 @@ mod tests { #[test] fn test_process_harvest_landsat_item_produces_correct_tiles() { - let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( - "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" - )) - .expect("valid Landsat mapping fixture"); + let api_mapping: crate::api::model::services::StacDataProviderDefinition = + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" + )) + .expect("valid Landsat mapping fixture"); + let mapping: StacDataProviderDefinition = api_mapping.into(); let items: stac::ItemCollection = serde_json::from_str(include_str!( "../../../../test_data/stac_responses/items/landsat-c2-l1-harvest-test.json" @@ -1389,7 +1377,6 @@ mod tests { time_start: None, time_end: None, bbox: None, - limit: None, geo_engine_url: String::new(), geo_engine_email: String::new(), geo_engine_password: String::new(), diff --git a/geoengine/services/src/contexts/migrations/current_schema.sql b/geoengine/services/src/contexts/migrations/current_schema.sql index 05296a4a2b..24d7cdfa0f 100644 --- a/geoengine/services/src/contexts/migrations/current_schema.sql +++ b/geoengine/services/src/contexts/migrations/current_schema.sql @@ -907,6 +907,7 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], + page_limit bigint, query_timeout_secs bigint ); diff --git a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql index cd9c408c63..e68600336f 100644 --- a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql +++ b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql @@ -29,6 +29,7 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], + page_limit bigint, query_timeout_secs bigint ); diff --git a/geoengine/services/src/datasets/external/stac/listing.rs b/geoengine/services/src/datasets/external/stac/listing.rs index 8a5ddc40e6..bab0da8537 100644 --- a/geoengine/services/src/datasets/external/stac/listing.rs +++ b/geoengine/services/src/datasets/external/stac/listing.rs @@ -527,6 +527,7 @@ mod tests { ), bands: vec![], }], + 100_i64, 60, ) } @@ -593,6 +594,7 @@ mod tests { bands: vec![], }, ], + 100_i64, 60, ) } diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 56264daece..9f4b3d9f3c 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -37,6 +37,7 @@ struct StacMultiBandMetaData { s3_config: Option, time_dimension: TimeDimension, dataset: StacProviderDataset, + page_limit: i64, client: reqwest::Client, /// Shared query-result cache from the provider. query_cache: Arc, @@ -371,7 +372,7 @@ impl StacMultiBandMetaData { .to_datetime_string_with_millis(), ), ), - ("limit".to_owned(), "100".to_owned()), + ("limit".to_owned(), self.page_limit.to_string()), ( "fields".to_owned(), "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform".to_owned(), @@ -796,6 +797,7 @@ impl s3_config: self.s3_config.clone(), time_dimension: self.time_dimension, dataset: dataset.clone(), + page_limit: self.page_limit, client: self.client.clone(), query_cache: self.query_cache.clone(), })) @@ -868,6 +870,7 @@ mod tests { }, ], }], + page_limit: 100, query_timeout_secs: 60, } } @@ -1104,6 +1107,7 @@ mod tests { ], }, ], + page_limit: 100, query_timeout_secs: 60, }; @@ -1150,13 +1154,16 @@ mod tests { /// harvester and the runtime provider. #[crate::ge_context::test] async fn mapping_from_discover_works_as_stacdataprovider(app_ctx: PostgresContext) { - // Load the discover-generated mapping JSON - let mut provider_def: crate::datasets::external::stac::StacDataProviderDefinition = + // Load the discover-generated mapping JSON via the API layer (camelCase) + let api_def: crate::api::model::services::StacDataProviderDefinition = serde_json::from_str(include_str!( "../../../../../test_data/stac_responses/expected-mapping-code-de.json" )) .expect("valid discover mapping fixture"); + let mut provider_def: crate::datasets::external::stac::StacDataProviderDefinition = + api_def.into(); + // Use a placeholder URL (no actual HTTP calls needed for meta_data registration) provider_def.api_url = "https://stac.test/v1".to_owned(); provider_def.id = DataProviderId::new(); diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index 161c32fb5b..f4dbb8cd30 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -33,7 +33,7 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, // TODO: should this be on dataset level? pub datasets: Vec, - // TODO: page limit(?) + pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, @@ -94,6 +94,7 @@ impl DataProviderDefinition for StacDataProviderDefinition { self.s3_config, self.time_dimension, self.datasets, + self.page_limit, self.query_timeout_secs, ))) } @@ -150,6 +151,7 @@ pub struct StacDataProvider { s3_config: Option, time_dimension: TimeDimension, datasets: Vec, + page_limit: i64, /// Shared HTTP client, reused across all requests for this provider. client: reqwest::Client, /// In-memory cache for STAC query results (tile files), keyed by dataset @@ -168,6 +170,7 @@ impl StacDataProvider { s3_config: Option, time_dimension: TimeDimension, datasets: Vec, + page_limit: i64, query_timeout_secs: i64, ) -> Self { let client = reqwest::Client::builder() @@ -183,6 +186,7 @@ impl StacDataProvider { s3_config, time_dimension, datasets, + page_limit, client, query_cache: Arc::new(StacQueryCache::default()), } diff --git a/geoengine/test_data/provider_defs_api/stac_sentinel2.json b/geoengine/test_data/provider_defs_api/stac_sentinel2.json index 6d8a2b513a..8835e3e080 100644 --- a/geoengine/test_data/provider_defs_api/stac_sentinel2.json +++ b/geoengine/test_data/provider_defs_api/stac_sentinel2.json @@ -413,5 +413,6 @@ } ] } - ] + ], + "pageLimit": 100 } diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json index 0e49bcb127..a7a0fe3296 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-code-de.json +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -1,4 +1,5 @@ { + "type": "StacProviderDefinition", "name": "sentinel-2-l2a from STAC", "id": "00000000-0000-0000-0000-000000000000", "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.test/v1", @@ -7,19 +8,18 @@ "collectionName": "sentinel-2-l2a", "s3Config": null, "timeDimension": { - "regular": { - "origin": 0, - "step": { "granularity": "days", "step": 1 } - } + "type": "regular", + "origin": 0, + "step": { "granularity": "days", "step": 1 } }, "datasets": [ { "name": "sentinel-2-l2a EPSG:32632 U16 10m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", + "dataType": "U16", "resolution": { "x": 10.0, "y": 10.0 }, "projection": "EPSG:32632", - "spatial_grid": { + "spatialGrid": { "spatialGrid": { "geoTransform": { "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, @@ -27,26 +27,26 @@ "yPixelSize": -10.0 }, "gridBounds": { - "min": [0, 0], - "max": [10979, 10979] + "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, + "bottomRightIdx": { "yIdx": 10979, "xIdx": 10979 } } }, - "state": "source" + "descriptor": "source" }, "bands": [ - { "asset_title": "Blue (band 2) - 10m", "band_name": null }, - { "asset_title": "Green (band 3) - 10m", "band_name": null }, - { "asset_title": "NIR 1 (band 8) - 10m", "band_name": null }, - { "asset_title": "Red (band 4) - 10m", "band_name": null } + { "assetTitle": "Blue (band 2) - 10m", "bandName": null }, + { "assetTitle": "Green (band 3) - 10m", "bandName": null }, + { "assetTitle": "NIR 1 (band 8) - 10m", "bandName": null }, + { "assetTitle": "Red (band 4) - 10m", "bandName": null } ] }, { "name": "sentinel-2-l2a EPSG:32632 U16 20m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", + "dataType": "U16", "resolution": { "x": 20.0, "y": 20.0 }, "projection": "EPSG:32632", - "spatial_grid": { + "spatialGrid": { "spatialGrid": { "geoTransform": { "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, @@ -54,16 +54,17 @@ "yPixelSize": -20.0 }, "gridBounds": { - "min": [0, 0], - "max": [5489, 5489] + "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, + "bottomRightIdx": { "yIdx": 5489, "xIdx": 5489 } } }, - "state": "source" + "descriptor": "source" }, "bands": [ - { "asset_title": "SWIR 1 (band 11) - 20m", "band_name": null }, - { "asset_title": "SWIR 2 (band 12) - 20m", "band_name": null } + { "assetTitle": "SWIR 1 (band 11) - 20m", "bandName": null }, + { "assetTitle": "SWIR 2 (band 12) - 20m", "bandName": null } ] } - ] + ], + "pageLimit": 100 } diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json index 05713a7a10..08d2f1a7ef 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -1,4 +1,5 @@ { + "type": "StacProviderDefinition", "name": "landsat-c2-l1 from STAC", "id": "00000000-0000-0000-0000-000000000000", "description": "Auto-discovered mapping for STAC collection 'landsat-c2-l1' at https://stac.test/v1", @@ -7,19 +8,18 @@ "collectionName": "landsat-c2-l1", "s3Config": null, "timeDimension": { - "regular": { - "origin": 0, - "step": { "granularity": "days", "step": 1 } - } + "type": "regular", + "origin": 0, + "step": { "granularity": "days", "step": 1 } }, "datasets": [ { "name": "landsat-c2-l1 EPSG:32632 U16 30m", "description": "Auto-discovered from STAC collection 'landsat-c2-l1'", - "data_type": "U16", + "dataType": "U16", "resolution": { "x": 30.0, "y": 30.0 }, "projection": "EPSG:32632", - "spatial_grid": { + "spatialGrid": { "spatialGrid": { "geoTransform": { "originCoordinate": { "x": 399960.0, "y": 5800020.0 }, @@ -27,18 +27,19 @@ "yPixelSize": -30.0 }, "gridBounds": { - "min": [0, 0], - "max": [7999, 7999] + "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, + "bottomRightIdx": { "yIdx": 7999, "xIdx": 7999 } } }, - "state": "source" + "descriptor": "source" }, "bands": [ - { "asset_title": "Blue (band 2) - 30m", "band_name": null }, - { "asset_title": "Green (band 3) - 30m", "band_name": null }, - { "asset_title": "NIR (band 5) - 30m", "band_name": null }, - { "asset_title": "Red (band 4) - 30m", "band_name": null } + { "assetTitle": "Blue (band 2) - 30m", "bandName": null }, + { "assetTitle": "Green (band 3) - 30m", "bandName": null }, + { "assetTitle": "NIR (band 5) - 30m", "bandName": null }, + { "assetTitle": "Red (band 4) - 30m", "bandName": null } ] } - ] + ], + "pageLimit": 100 } From 28df5934cdcafdfd80c52f80f3d112ddf01711dc Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 18:53:14 +0000 Subject: [PATCH 6/9] refactors and fixes --- geoengine/services/src/api/model/services.rs | 38 +- .../src/cli/stac_harvester/discover.rs | 328 ++++++++--- .../src/cli/stac_harvester/harvest.rs | 136 ++--- .../services/src/cli/stac_harvester/mod.rs | 43 +- .../contexts/migrations/current_schema.sql | 7 +- .../migration_0029_stac_provider_band_name.rs | 31 + ...migration_0029_stac_provider_band_name.sql | 100 ++++ .../services/src/contexts/migrations/mod.rs | 3 + .../src/datasets/external/stac/common.rs | 171 ++++-- .../datasets/external/stac/loading_info.rs | 80 ++- .../src/datasets/external/stac/mod.rs | 55 +- geoengine/services/src/datasets/upload.rs | 7 + geoengine/services/src/util/mod.rs | 1 + geoengine/services/src/util/retry.rs | 187 ++++++ .../provider_defs_api/stac_sentinel2.json | 548 +++++++++++++++--- .../expected-mapping-code-de.json | 126 +++- .../expected-mapping-landsat-c2-l1.json | 80 ++- openapi.json | 40 +- 18 files changed, 1618 insertions(+), 363 deletions(-) create mode 100644 geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs create mode 100644 geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql create mode 100644 geoengine/services/src/util/retry.rs diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 922fa36342..9b4f19a240 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -989,13 +989,23 @@ impl From for StacProvide #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] -pub struct StacProviderDatasetBand { +pub struct StacAssetBand { pub asset_title: String, pub band_name: Option, } -impl From for crate::datasets::external::stac::StacProviderDatasetBand { - fn from(value: StacProviderDatasetBand) -> Self { +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct StacProviderDatasetBand { + /// The band inside the STAC asset that this dataset band reads from + /// (addressing: which asset file + which raster channel within it). + pub asset_band: StacAssetBand, + /// The band descriptor of the resulting geo engine dataset layer. + pub band_descriptor: crate::api::model::operators::RasterBandDescriptor, +} + +impl From for crate::datasets::external::stac::StacAssetBand { + fn from(value: StacAssetBand) -> Self { Self { asset_title: value.asset_title, band_name: value.band_name, @@ -1003,8 +1013,8 @@ impl From for crate::datasets::external::stac::StacProv } } -impl From for StacProviderDatasetBand { - fn from(value: crate::datasets::external::stac::StacProviderDatasetBand) -> Self { +impl From for StacAssetBand { + fn from(value: crate::datasets::external::stac::StacAssetBand) -> Self { Self { asset_title: value.asset_title, band_name: value.band_name, @@ -1012,6 +1022,24 @@ impl From for StacProv } } +impl From for crate::datasets::external::stac::StacProviderDatasetBand { + fn from(value: StacProviderDatasetBand) -> Self { + Self { + asset_band: value.asset_band.into(), + band_descriptor: value.band_descriptor.into(), + } + } +} + +impl From for StacProviderDatasetBand { + fn from(value: crate::datasets::external::stac::StacProviderDatasetBand) -> Self { + Self { + asset_band: value.asset_band.into(), + band_descriptor: value.band_descriptor.into(), + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] pub struct StacProviderDataset { diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index dc35b68e0e..3dc9f80304 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -12,12 +12,11 @@ use geoengine_datatypes::{ use ordered_float::OrderedFloat; use tracing::{info, warn}; -use crate::api::model::datatypes::{Measurement, UnitlessMeasurement, UnitlessMeasurementTypeTag}; -use crate::api::model::operators::RasterBandDescriptor; use crate::datasets::external::stac::{ - StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, StacProviderS3Config, - common, + StacAssetBand, StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, + StacProviderS3Config, common, }; +use crate::util::retry::{RetryPolicy, retry_http}; use geoengine_datatypes::primitives::{ RegularTimeDimension as DtRegularTimeDimension, TimeDimension as DtTimeDimension, }; @@ -88,6 +87,14 @@ pub struct StacDiscoverMapping { #[arg(long, default_value_t = false)] pub full_projection_grid: bool, + /// Provider id to write into the output definition. + /// + /// Discovery normally assigns a fresh random id. Pass the `id` of the + /// existing mapping file when regenerating it so that provider-scoped + /// references (e.g. `_::...` in layer bodies) stay valid. + #[arg(long)] + pub id: Option, + /// Time dimension granularity (default: days) #[arg(long, default_value = "days")] pub time_granularity: String, @@ -121,9 +128,10 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), params.stac_collection ); - let collection: stac::Collection = stac_api_request_parse(&client, &collection_url) - .await - .context("Failed to fetch STAC collection")?; + let collection: stac::Collection = + stac_api_request_parse(&client, &collection_url, &stac_request_policy()) + .await + .context("Failed to fetch STAC collection")?; let dataset_bands = scan_collection_bands(&collection, ¶ms.file_types); @@ -176,7 +184,7 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), let provider_def = StacDataProviderDefinition { name: format!("{} from STAC", params.stac_collection), - id: DataProviderId::new(), + id: params.id.unwrap_or_else(DataProviderId::new), description: format!( "Auto-discovered mapping for STAC collection '{}' at {}", params.stac_collection, params.stac_url @@ -188,13 +196,33 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), time_dimension, datasets, page_limit: params.page_limit as i64, + query_timeout_secs: 60, }; - let json = serde_json::to_string_pretty( - &crate::api::model::services::StacDataProviderDefinition::from(provider_def), + let s3_config = provider_def.s3_config.clone(); + + let mut json_value = serde_json::to_value( + crate::api::model::services::StacDataProviderDefinition::from(provider_def), ) .context("Failed to serialize mapping to JSON")?; + // The API model wraps S3 credentials in `Secret`, which serializes as + // `*****`. Keep the plain values (e.g. `__AWS_ACCESS_KEY_ID__` markers) in + // the output file instead, so they can be substituted at runtime. + if let (Some(obj), Some(s3)) = (json_value.as_object_mut(), &s3_config) + && let Some(s3_obj) = obj.get_mut("s3Config").and_then(|v| v.as_object_mut()) + { + if let Some(key) = &s3.access_key { + s3_obj.insert("accessKey".to_string(), serde_json::json!(key)); + } + if let Some(key) = &s3.secret_key { + s3_obj.insert("secretKey".to_string(), serde_json::json!(key)); + } + } + + let json = + serde_json::to_string_pretty(&json_value).context("Failed to serialize mapping to JSON")?; + if let Some(output_path) = ¶ms.output { std::fs::write(output_path, &json) .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; @@ -221,8 +249,9 @@ struct DiscoveredDatasetInfo { fn scan_collection_bands( collection: &stac::Collection, file_types: &[ImportFileType], -) -> HashMap> { - let mut dataset_bands: HashMap> = HashMap::new(); +) -> HashMap> { + let mut dataset_bands: HashMap> = + HashMap::new(); for (_asset_key, asset) in &collection.item_assets { if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { @@ -275,11 +304,11 @@ async fn fetch_sample_items( loop { let page = if let Some(ref url) = next_url { - stac_api_request_parse::(client, url) + stac_api_request_parse::(client, url, &stac_request_policy()) .await .context("Failed to fetch sample items page")? } else { - stac_api_request_with_params(client, &base_url, &query_params) + stac_api_request_with_params(client, &base_url, &query_params, &stac_request_policy()) .await .context("Failed to fetch sample items")? }; @@ -364,13 +393,17 @@ fn process_sample_assets( }; let asset_title = asset.title.as_deref().unwrap_or(asset_key).to_string(); - let band_names = common::band_names_from_asset_v1_1_0(asset) - .unwrap_or_else(|_| vec![asset_title.clone()]); + let asset_info = common::band_names_from_asset_v1_1_0(asset).unwrap_or_else(|_| { + common::AssetBandInfo { + asset_title: asset_title.clone(), + band_names: vec![asset_title.clone()], + } + }); let entry = sample_band_info.entry(partial_key.clone()).or_default(); - for bn in &band_names { - if !entry.iter().any(|(t, _)| t == &asset_title) { - entry.push((asset_title.clone(), bn.clone())); + for bn in &asset_info.band_names { + if !entry.iter().any(|(t, _)| t == &asset_info.asset_title) { + entry.push((asset_info.asset_title.clone(), bn.clone())); } } @@ -394,7 +427,7 @@ fn process_sample_assets( /// and sample band information. fn build_datasets( discovered_datasets: &HashMap, - dataset_bands: &HashMap>, + dataset_bands: &HashMap>, sample_band_info: &HashMap>, full_projection_grid: bool, stac_collection: &str, @@ -408,25 +441,35 @@ fn build_datasets( }; let mut bands: Vec = Vec::new(); + // Track the *resolved* band name (what the harvest uses: + // `band_name`, falling back to the asset title). Raster band names must + // be unique, so skip any band whose resolved name is already present. + let mut seen_names: std::collections::HashSet = std::collections::HashSet::new(); // Use bands from collection-level scan if let Some(descriptors) = dataset_bands.get(&partial_key) { - for desc in descriptors { - bands.push(StacProviderDatasetBand { - asset_title: desc.name.clone(), - band_name: None, - }); + for band in descriptors { + let resolved = band + .asset_band + .band_name + .clone() + .unwrap_or_else(|| band.asset_band.asset_title.clone()); + if seen_names.insert(resolved) { + bands.push(band.clone()); + } } } // Enrich with sample item band info if let Some(sample_bands) = sample_band_info.get(&partial_key) { for (asset_title, band_name) in sample_bands { - if !bands.iter().any(|b| b.asset_title == *asset_title) { - bands.push(StacProviderDatasetBand { + // Sample bands always carry an explicit band name; skip any + // whose resolved name is already present. + if seen_names.insert(band_name.clone()) { + bands.push(StacProviderDatasetBand::new_unitless(StacAssetBand { asset_title: asset_title.clone(), band_name: Some(band_name.clone()), - }); + })); } } } @@ -436,7 +479,7 @@ fn build_datasets( continue; } - bands.sort_by(|a, b| a.asset_title.cmp(&b.asset_title)); + bands.sort_by(|a, b| a.asset_band.asset_title.cmp(&b.asset_band.asset_title)); let spatial_grid = build_dataset_spatial_grid(info, dataset_key, full_projection_grid); @@ -604,7 +647,7 @@ fn scan_collection_item_asset( collection_version: &stac::Version, asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, -) -> Result>>, String> { +) -> Result>>, String> { match collection_version { stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset), stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries), @@ -619,8 +662,9 @@ fn scan_collection_item_asset( fn scan_collection_item_asset_v1_0_0( asset: &stac::ItemAsset, -) -> Result>>, String> { - let mut dataset_bands: HashMap> = HashMap::new(); +) -> Result>>, String> { + let mut dataset_bands: HashMap> = + HashMap::new(); let Some(raster_bands) = asset.additional_fields.get("raster:bands") else { return Ok(None); @@ -644,6 +688,8 @@ fn scan_collection_item_asset_v1_0_0( return Ok(None); } + let asset_title = asset.title.clone().unwrap_or_default(); + for (index, raster_band) in raster_bands.into_iter().enumerate() { let data_type = raster_band .data_type @@ -671,12 +717,10 @@ fn scan_collection_item_asset_v1_0_0( resolution, }) .or_default() - .push(RasterBandDescriptor { - name: band_name, - measurement: Measurement::Unitless(UnitlessMeasurement { - r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, - }), - }); + .push(StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: asset_title.clone(), + band_name: Some(band_name), + })); } Ok(Some(dataset_bands)) @@ -685,8 +729,9 @@ fn scan_collection_item_asset_v1_0_0( fn scan_collection_item_asset_v1_1_0( asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, -) -> Result>>, String> { - let mut dataset_bands: HashMap> = HashMap::new(); +) -> Result>>, String> { + let mut dataset_bands: HashMap> = + HashMap::new(); let data_type = asset .additional_fields @@ -698,7 +743,7 @@ fn scan_collection_item_asset_v1_1_0( let raster_data_type = common::raster_data_type_from_stac_data_type_str(data_type) .ok_or_else(|| format!("Unsupported data_type: {data_type}"))?; - let band_names = common::band_names_from_item_asset_v1_1_0(asset)?; + let asset_info = common::band_names_from_item_asset_v1_1_0(asset)?; let resolution = asset .additional_fields @@ -717,28 +762,23 @@ fn scan_collection_item_asset_v1_1_0( }) .ok_or_else(|| "Missing attribute `gsd` or `proj:transform`".to_string())?; - for band_name in band_names { - dataset_bands - .entry(PartialDatasetKey { - data_type: raster_data_type, - resolution: resolution.into(), - }) - .or_default() - .push(RasterBandDescriptor { - name: band_name.clone(), - measurement: Measurement::Unitless(UnitlessMeasurement { - r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, - }), - }); + let entry = dataset_bands + .entry(PartialDatasetKey { + data_type: raster_data_type, + resolution: resolution.into(), + }) + .or_default(); + for band_name in asset_info.band_names { + entry.push(StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: asset_info.asset_title.clone(), + band_name: Some(band_name), + })); } Ok(Some(dataset_bands)) } -fn matches_selected_file_types( - media_type: Option<&str>, - file_types: &[ImportFileType], -) -> bool { +fn matches_selected_file_types(media_type: Option<&str>, file_types: &[ImportFileType]) -> bool { file_types.iter().any(|file_type| match file_type { ImportFileType::Cog => common::is_cog_media_type(media_type), ImportFileType::Jp2 => common::is_jp2_media_type(media_type), @@ -746,14 +786,26 @@ fn matches_selected_file_types( } fn merge_dataset_bands( - dataset_bands: &mut HashMap>, - additions: HashMap>, + dataset_bands: &mut HashMap>, + additions: HashMap>, ) { - for (partial_key, band_descriptors) in additions { + for (partial_key, bands) in additions { let existing_bands = dataset_bands.entry(partial_key).or_default(); - for descriptor in band_descriptors { - if existing_bands.iter().all(|b| b.name != descriptor.name) { - existing_bands.push(descriptor); + for band in bands { + let resolved = band + .asset_band + .band_name + .clone() + .unwrap_or_else(|| band.asset_band.asset_title.clone()); + let already_present = existing_bands.iter().any(|b| { + b.asset_band + .band_name + .clone() + .unwrap_or_else(|| b.asset_band.asset_title.clone()) + == resolved + }); + if !already_present { + existing_bands.push(band); } } } @@ -784,21 +836,25 @@ fn parse_time_dimension(granularity: &str, step: u64) -> Result RetryPolicy { + RetryPolicy::new().stop_on_status(&[400, 404]) +} + async fn stac_api_request_parse( client: &reqwest::Client, url: &str, + policy: &RetryPolicy, ) -> Result { - let response = client - .get(url) - .send() - .await - .with_context(|| format!("Failed to fetch {url}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("STAC API returned HTTP {status}: {body}"); - } + let response = retry_http( + || async { client.get(url).send().await?.error_for_status() }, + &format!("Fetch {url}"), + policy, + |e| e.status().map(|s| s.as_u16()), + ) + .await + .with_context(|| format!("Failed to fetch {url}"))?; response .json() @@ -810,19 +866,23 @@ async fn stac_api_request_with_params( client: &reqwest::Client, url: &str, params: &[(String, String)], + policy: &RetryPolicy, ) -> Result { - let response = client - .get(url) - .query(params) - .send() - .await - .with_context(|| format!("Failed to fetch {url}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("STAC API returned HTTP {status}: {body}"); - } + let response = retry_http( + || async { + client + .get(url) + .query(params) + .send() + .await? + .error_for_status() + }, + &format!("Fetch {url}"), + policy, + |e| e.status().map(|s| s.as_u16()), + ) + .await + .with_context(|| format!("Failed to fetch {url}"))?; response .json() @@ -899,6 +959,7 @@ mod tests { time_granularity: "days".to_string(), time_step: 1, page_limit: 100, + id: None, }; discover_mapping(params) @@ -1010,6 +1071,7 @@ mod tests { time_granularity: "days".to_string(), time_step: 1, page_limit: 100, + id: None, }; discover_mapping(params) @@ -1049,4 +1111,96 @@ mod tests { let _ = std::fs::remove_file(&output_path); } + + // ----------------------------------------------------------------------- + // build_datasets band dedup + // ----------------------------------------------------------------------- + + fn unitless_band_descriptor(asset_title: &str, band_name: &str) -> StacProviderDatasetBand { + StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: asset_title.to_string(), + band_name: Some(band_name.to_string()), + }) + } + + #[test] + fn test_build_datasets_dedupes_duplicate_resolved_band_names() { + let partial_key = PartialDatasetKey { + data_type: RasterDataType::U8, + resolution: OrderedFloat(10.0), + }; + let dataset_key = DatasetKey { + epsg: 32_632, + data_type: RasterDataType::U8, + resolution: OrderedFloat(10.0), + }; + + // Collection-level scan reports the true-color asset as one dataset + // band per STAC band, keeping the real asset title separate from the + // band name (no `True color image [B02]`-style synthetic titles). + let mut dataset_bands = HashMap::new(); + dataset_bands.insert( + partial_key.clone(), + vec![ + unitless_band_descriptor("True color image", "B02"), + unitless_band_descriptor("True color image", "B03"), + unitless_band_descriptor("True color image", "B04"), + ], + ); + + // Sample items report the same true-color asset with an explicit band + // name that would collide with the collection scan's `B04` band. + let mut sample_band_info = HashMap::new(); + sample_band_info.insert( + partial_key.clone(), + vec![("True color image".to_string(), "B04".to_string())], + ); + + let mut discovered_datasets = HashMap::new(); + discovered_datasets.insert( + dataset_key, + DiscoveredDatasetInfo { + geo_transform: Some(GeoTransform::new((0.0, 0.0).into(), 10.0, -10.0)), + proj_shape: Some((100, 100)), + srs: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32_632), + asset_count: 1, + }, + ); + + let datasets = build_datasets( + &discovered_datasets, + &dataset_bands, + &sample_band_info, + false, + "sentinel-2-l2a", + ); + + assert_eq!(datasets.len(), 1); + + let names: Vec = datasets[0] + .bands + .iter() + .map(|b| { + b.asset_band + .band_name + .clone() + .unwrap_or_else(|| b.asset_band.asset_title.clone()) + }) + .collect(); + + // The duplicate `B04` from the sample item must be dropped so that all + // resolved band names are unique. + assert_eq!(names, vec!["B02", "B03", "B04"]); + + // All bands keep the real asset title. + assert!( + datasets[0] + .bands + .iter() + .all(|b| b.asset_band.asset_title == "True color image") + ); + + let unique: std::collections::HashSet<_> = names.iter().collect(); + assert_eq!(unique.len(), names.len()); + } } diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 9b57c259bf..06c532fe89 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -1,9 +1,4 @@ -use std::{ - collections::HashMap, - io::Read, - str::FromStr, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, io::Read, str::FromStr, time::Instant}; use anyhow::Context; use chrono::Timelike; @@ -19,6 +14,7 @@ use tracing::{debug, error, info, warn}; use crate::datasets::external::stac::{ StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, common, }; +use crate::util::retry::{RetryPolicy, retry_http}; use crate::{ api::{ handlers::{ @@ -32,8 +28,8 @@ use crate::{ model::{ datatypes::{ GdalConfigOption, GridBoundingBox2D as ApiGridBoundingBox2D, - GridIdx2D as ApiGridIdx2D, LayerId, Measurement, SpatialGridDefinition, - TimeGranularity, TimeStep, UnitlessMeasurement, UnitlessMeasurementTypeTag, + GridIdx2D as ApiGridIdx2D, LayerId, SpatialGridDefinition, TimeGranularity, + TimeStep, }, operators::{ GdalDatasetParameters, GdalMultiBand, GdalMultiBandTypeTag, RasterBandDescriptor, @@ -55,11 +51,11 @@ use crate::{ permissions::{Permission, Role}, workflows::workflow::Workflow, }; +use geoengine_api_client::apis::configuration::Configuration as ApiConfig; use geoengine_operators::{ engine::{RasterOperator, TypedOperator}, source::{MultiBandGdalSource, MultiBandGdalSourceParameters}, }; -use geoengine_api_client::apis::configuration::Configuration as ApiConfig; // --------------------------------------------------------------------------- // Harvest @@ -197,13 +193,7 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err upload_tiles_to_datasets(&api_config, ¶ms, &tiles_by_dataset).await?; - create_harvest_layer_collections( - &api_config, - provider_def, - &created_datasets, - ¶ms, - ) - .await?; + create_harvest_layer_collections(&api_config, provider_def, &created_datasets, ¶ms).await?; let elapsed = start_time.elapsed(); info!("Harvest completed in {:.2?}", elapsed); @@ -309,20 +299,39 @@ async fn dataset_exists_api( api_config: &ApiConfig, dataset_name: &str, ) -> Result { + // A missing dataset is signalled with HTTP 400 (`CannotLoadDataset`) and + // conventionally 404 — a definitive "doesn't exist" answer. Configure the + // retry policy to stop on those codes so they are not retried. + let policy = RetryPolicy::new().stop_on_status(&[400, 404]); + let result = retry_http( || geoengine_api_client::apis::datasets_api::get_dataset_handler(api_config, dataset_name), &format!("Check dataset existence for '{dataset_name}'"), + &policy, + apis_error_status, ) .await; match result { Ok(_) => Ok(true), Err(geoengine_api_client::apis::Error::ResponseError(resp)) - if resp.status == reqwest::StatusCode::BAD_REQUEST => + if resp.status == reqwest::StatusCode::BAD_REQUEST + || resp.status == reqwest::StatusCode::NOT_FOUND => { Ok(false) } - Err(e) => Err(anyhow::anyhow!("Failed to check dataset '{dataset_name}': {e}")), + Err(e) => Err(anyhow::anyhow!( + "Failed to check dataset '{dataset_name}': {e}" + )), + } +} + +/// Extract an HTTP status code from a `geoengine_api_client` API error, so a +/// [`RetryPolicy`] can match on it. +fn apis_error_status(e: &geoengine_api_client::apis::Error) -> Option { + match e { + geoengine_api_client::apis::Error::ResponseError(resp) => Some(resp.status.as_u16()), + _ => None, } } @@ -339,12 +348,7 @@ async fn create_dataset_api( let bands: Vec = dataset .bands .iter() - .map(|b| RasterBandDescriptor { - name: b.band_name.clone().unwrap_or_else(|| b.asset_title.clone()), - measurement: Measurement::Unitless(UnitlessMeasurement { - r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, - }), - }) + .map(|b| b.band_descriptor.clone().into()) .collect(); let dt_gt: GeoTransform = dataset.spatial_grid.geo_transform(); @@ -408,6 +412,8 @@ async fn create_dataset_api( .await }, &format!("Create dataset '{dataset_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -469,6 +475,8 @@ async fn share_dataset_api( .await }, &format!("Add permission for dataset '{dataset_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; } @@ -551,6 +559,8 @@ async fn create_harvest_layer_collections( .await }, &format!("Create layer '{layer_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -571,9 +581,7 @@ async fn create_layer_collection_api( let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); let client = &api_config.client; - if let Some(existing_id) = - find_child_collection_by_name(api_config, parent_id, name).await? - { + if let Some(existing_id) = find_child_collection_by_name(api_config, parent_id, name).await? { if params.verbose { info!("Found existing layer collection '{name}'"); } @@ -601,6 +609,8 @@ async fn create_layer_collection_api( .await }, &format!("Create layer collection '{name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -636,6 +646,8 @@ async fn find_child_collection_by_name( .await }, &format!("List child collections of {parent_id}"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -694,6 +706,8 @@ async fn share_layer_collection_api( .await }, &format!("Share collection with role {}", permission.role_id), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; } @@ -701,10 +715,7 @@ async fn share_layer_collection_api( Ok(()) } -async fn share_layer_api( - api_config: &ApiConfig, - layer_id: &LayerId, -) -> Result<(), anyhow::Error> { +async fn share_layer_api(api_config: &ApiConfig, layer_id: &LayerId) -> Result<(), anyhow::Error> { let geo_engine_url = &api_config.base_path; let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); let client = &api_config.client; @@ -740,6 +751,8 @@ async fn share_layer_api( .await }, &format!("Share layer with role {}", permission.role_id), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; } @@ -751,9 +764,6 @@ async fn share_layer_api( // Pagination // --------------------------------------------------------------------------- -const MAX_RETRIES: u32 = 10; -const INITIAL_RETRY_DELAY_MS: u64 = 1000; - #[derive(Debug, Clone)] enum QueryState { FirstPage { @@ -823,6 +833,8 @@ async fn query_item_collection_internal( .await }, "Query STAC first page", + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -840,6 +852,8 @@ async fn query_item_collection_internal( let item_collection: stac::ItemCollection = retry_http( || async { client.get(next_url).send().await?.json().await }, "Query STAC next page", + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -857,36 +871,6 @@ async fn query_item_collection_internal( } } -// --------------------------------------------------------------------------- -// HTTP retry helper -// --------------------------------------------------------------------------- - -async fn retry_http(mut operation: F, operation_name: &str) -> Result -where - F: FnMut() -> Fut, - Fut: std::future::Future>, - E: std::fmt::Display, -{ - let mut attempt = 0; - loop { - match operation().await { - Ok(result) => return Ok(result), - Err(err) => { - attempt += 1; - if attempt >= MAX_RETRIES { - error!("{operation_name} failed after {MAX_RETRIES} attempts: {err}"); - return Err(err); - } - let delay = Duration::from_millis(INITIAL_RETRY_DELAY_MS * 2_u64.pow(attempt - 1)); - warn!( - "{operation_name} failed (attempt {attempt}/{MAX_RETRIES}): {err}. Retrying in {delay:?}..." - ); - tokio::time::sleep(delay).await; - } - } - } -} - // --------------------------------------------------------------------------- // Authentication helper // --------------------------------------------------------------------------- @@ -911,6 +895,8 @@ async fn create_api_config( let session = retry_http( || geoengine_api_client::apis::session_api::login_handler(&config, credentials.clone()), "Login to Geo Engine", + &RetryPolicy::new(), + apis_error_status, ) .await .context("Failed to authenticate")?; @@ -941,13 +927,7 @@ async fn setup_datasets( } if !dataset_exists_api(api_config, &dataset_name).await? { - create_dataset_api( - api_config, - &dataset_name, - dataset, - ¶ms.volume_name, - ) - .await?; + create_dataset_api(api_config, &dataset_name, dataset, ¶ms.volume_name).await?; created_datasets.push((idx, dataset.clone())); } } @@ -1095,9 +1075,7 @@ async fn upload_tiles_to_datasets( let response = retry_http( || async { client - .post(format!( - "{geo_engine_url}/dataset/{dataset_name}/tiles", - )) + .post(format!("{geo_engine_url}/dataset/{dataset_name}/tiles")) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {session_id}")) .json(chunk) @@ -1105,6 +1083,8 @@ async fn upload_tiles_to_datasets( .await }, &format!("Add tiles to dataset '{dataset_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; @@ -1139,7 +1119,7 @@ fn try_create_tile_for_band( let (_asset_key, asset) = item .assets .iter() - .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title))?; + .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_band.asset_title))?; // Check data type matches if let Some(asset_dt) = common::data_type_from_asset_v1_1_0_fallback(asset) @@ -1179,8 +1159,10 @@ fn try_create_tile_for_band( let (height, width) = common::proj_shape_from_fields(&asset.additional_fields)?; - let rasterband_channel = - common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref())?; + let rasterband_channel = common::rasterband_channel_for_dataset_band( + asset, + band_def.asset_band.band_name.as_deref(), + )?; let grid_bounds = GridBoundingBox2D::new( GridIdx2D::new([0, 0]), diff --git a/geoengine/services/src/cli/stac_harvester/mod.rs b/geoengine/services/src/cli/stac_harvester/mod.rs index 7ab60a16f6..430cd0f3db 100644 --- a/geoengine/services/src/cli/stac_harvester/mod.rs +++ b/geoengine/services/src/cli/stac_harvester/mod.rs @@ -19,6 +19,11 @@ pub use discover::StacDiscoverMapping; pub use harvest::StacHarvest; use clap::{Parser, Subcommand}; +use tracing_subscriber::{ + filter::{LevelFilter, Targets}, + layer::{Layer, SubscriberExt}, + util::SubscriberInitExt, +}; /// STAC harvester for Geo Engine #[derive(Debug, Parser)] @@ -40,8 +45,44 @@ pub enum StacHarvesterCommand { pub async fn stac_harvester(params: StacHarvester) -> Result<(), anyhow::Error> { match params.command { StacHarvesterCommand::DiscoverMapping(discover) => { + init_stac_harvest_logging(discover.verbose); discover::discover_mapping(*discover).await } - StacHarvesterCommand::Harvest(harvest) => harvest::harvest_tiles(*harvest).await, + StacHarvesterCommand::Harvest(harvest) => { + init_stac_harvest_logging(harvest.verbose); + harvest::harvest_tiles(*harvest).await + } } } + +/// Initialize the tracing subscriber for STAC harvesting. +/// +/// Logs go to stderr so stdout stays clean (the `discover-mapping` command +/// emits the mapping JSON on stdout). Only the Geo Engine crates are logged — +/// dependency noise (hyper, reqwest, …) is filtered out. `--verbose` raises +/// the level to DEBUG; the default level is INFO so the progress `info!` +/// messages are visible. +fn init_stac_harvest_logging(verbose: bool) { + let geoengine_level = if verbose { + LevelFilter::DEBUG + } else { + LevelFilter::INFO + }; + + // `Targets` matches target prefixes hierarchically, so `geoengine_services` + // also covers `geoengine_services::cli::stac_harvester::harvest`. + let targets = Targets::new() + .with_target("geoengine_services", geoengine_level) + .with_target("geoengine_operators", geoengine_level) + .with_target("geoengine_datatypes", geoengine_level) + .with_target("geoengine_api_client", geoengine_level) + .with_default(LevelFilter::OFF); + + let _ = tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_writer(std::io::stderr) + .with_filter(targets), + ) + .try_init(); +} diff --git a/geoengine/services/src/contexts/migrations/current_schema.sql b/geoengine/services/src/contexts/migrations/current_schema.sql index 24d7cdfa0f..4f3f26d53e 100644 --- a/geoengine/services/src/contexts/migrations/current_schema.sql +++ b/geoengine/services/src/contexts/migrations/current_schema.sql @@ -882,11 +882,16 @@ CREATE TYPE "StacProviderS3Config" AS ( secret_key text ); -CREATE TYPE "StacProviderDatasetBand" AS ( +CREATE TYPE "StacAssetBand" AS ( asset_title text, band_name text ); +CREATE TYPE "StacProviderDatasetBand" AS ( + asset_band "StacAssetBand", + band_descriptor "RasterBandDescriptor" +); + CREATE TYPE "StacProviderDataset" AS ( "name" text, description text, diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs new file mode 100644 index 0000000000..330a23f7d6 --- /dev/null +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs @@ -0,0 +1,31 @@ +use super::database_migration::{DatabaseVersion, Migration}; +use crate::{ + contexts::migrations::migration_0028_stac_provider::Migration0028StacProvider, error::Result, +}; +use async_trait::async_trait; +use tokio_postgres::Transaction; + +/// This migration bundles the band addressing fields of `StacProviderDatasetBand` +/// into a nested `StacAssetBand` type and adds a `name` attribute of type +/// `RasterBandDescriptor` for the band in the resulting geo engine dataset +/// layer, independent of `asset_title`/`band_name`, which address the band +/// inside the STAC asset files. +pub struct Migration0029StacProviderBandName; + +#[async_trait] +impl Migration for Migration0029StacProviderBandName { + fn prev_version(&self) -> Option { + Some(Migration0028StacProvider.version()) + } + + fn version(&self) -> DatabaseVersion { + "0029_stac_provider_band_name".into() + } + + async fn migrate(&self, tx: &Transaction<'_>) -> Result<()> { + tx.batch_execute(include_str!("migration_0029_stac_provider_band_name.sql")) + .await?; + + Ok(()) + } +} diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql new file mode 100644 index 0000000000..55dde1e39a --- /dev/null +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -0,0 +1,100 @@ +-- Bundle the band *addressing* fields into a nested `StacAssetBand` type and +-- add a `band_descriptor` (`RasterBandDescriptor`) for the band in the +-- resulting geo engine dataset layer. +-- +-- Before: "StacProviderDatasetBand" (asset_title text, band_name text) +-- After: "StacAssetBand" (asset_title text, band_name text) +-- "StacProviderDatasetBand" (asset_band "StacAssetBand", +-- band_descriptor "RasterBandDescriptor") +-- +-- `asset_title`/`band_name` *address* a band inside the STAC asset files (which +-- asset file, and which raster channel within it). `band_descriptor` is the +-- `RasterBandDescriptor` of the geo engine dataset layer band, populated with +-- the naming fallback ("use band_name, then asset_title") and a unitless +-- measurement. + +CREATE TYPE "StacAssetBand" AS ( + asset_title text, + band_name text +); + +ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE asset_band "StacAssetBand"; +ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE band_descriptor "RasterBandDescriptor"; + +-- Migrate existing stored STAC provider definitions: bundle the flat +-- `asset_title`/`band_name` attributes into the new `asset_band` attribute. +CREATE FUNCTION pg_temp.stac_migrate_bands( + bands "StacProviderDatasetBand" [] +) RETURNS "StacProviderDatasetBand" [] AS $$ +DECLARE + b "StacProviderDatasetBand"; + new_bands "StacProviderDatasetBand" [] := '{}'; +BEGIN + FOREACH b IN ARRAY bands LOOP + new_bands := array_append( + new_bands, + ROW( + NULL, + NULL, + ROW((b).asset_title, (b).band_name)::"StacAssetBand", + ROW( + COALESCE((b).band_name, (b).asset_title), + ROW(NULL, NULL)::"Measurement" + )::"RasterBandDescriptor" + )::"StacProviderDatasetBand" + ); + END LOOP; + RETURN new_bands; +END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION pg_temp.stac_migrate_provider_def( + def "StacDataProviderDefinition" +) RETURNS "StacDataProviderDefinition" AS $$ +DECLARE + d "StacProviderDataset"; + new_datasets "StacProviderDataset" [] := '{}'; +BEGIN + FOREACH d IN ARRAY (def).datasets LOOP + new_datasets := array_append( + new_datasets, + ROW( + (d).name, + (d).description, + (d).data_type, + (d).resolution, + (d).projection, + (d).spatial_grid, + pg_temp.stac_migrate_bands((d).bands) + )::"StacProviderDataset" + ); + END LOOP; + RETURN ROW( + (def).name, + (def).id, + (def).description, + (def).priority, + (def).api_url, + (def).collection_name, + (def).s3_config, + (def).time_dimension, + new_datasets, + (def).page_limit, + (def).query_timeout_secs + )::"StacDataProviderDefinition"; +END; +$$ LANGUAGE plpgsql; + +UPDATE layer_providers +SET definition = ROW( + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pg_temp.stac_migrate_provider_def((definition).stac_data_provider_definition) +)::"DataProviderDefinition" +WHERE (definition).stac_data_provider_definition IS NOT NULL; + +-- Drop the now-redundant flat addressing attributes. +ALTER TYPE "StacProviderDatasetBand" DROP ATTRIBUTE asset_title; +ALTER TYPE "StacProviderDatasetBand" DROP ATTRIBUTE band_name; + +DROP FUNCTION pg_temp.stac_migrate_bands; +DROP FUNCTION pg_temp.stac_migrate_provider_def; diff --git a/geoengine/services/src/contexts/migrations/mod.rs b/geoengine/services/src/contexts/migrations/mod.rs index 5ef9d6773b..400213effc 100644 --- a/geoengine/services/src/contexts/migrations/mod.rs +++ b/geoengine/services/src/contexts/migrations/mod.rs @@ -13,6 +13,7 @@ pub use crate::contexts::migrations::{ migration_0025_time_descriptor::Migration0025TimeDescriptor, migration_0027_tile_z_index::Migration0027TileZIndex, migration_0028_stac_provider::Migration0028StacProvider, + migration_0029_stac_provider_band_name::Migration0029StacProviderBandName, }; pub use database_migration::{ DatabaseVersion, Migration, MigrationResult, initialize_database, migrate_database, @@ -33,6 +34,7 @@ mod migration_0025_time_descriptor; mod migration_0026_gdal_tiles; mod migration_0027_tile_z_index; mod migration_0028_stac_provider; +mod migration_0029_stac_provider_band_name; #[cfg(test)] mod schema_info; @@ -67,6 +69,7 @@ pub fn all_migrations() -> Vec> { Box::new(Migration0026GdalTiles), Box::new(Migration0027TileZIndex), Box::new(Migration0028StacProvider), + Box::new(Migration0029StacProviderBandName), ] } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index ed175ef630..e537bf6244 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -48,6 +48,12 @@ pub fn geo_transform_from_fields( return None; } + // A `GeoTransform` requires non-zero pixel sizes. Some STAC catalogs encode + // angular/QA assets with a zero pixel height, so skip those instead of panicking. + if values[0] == 0.0 || values[4] == 0.0 { + return None; + } + // GDAL geo-transform: [origin_x, pixel_width, rotation, origin_y, rotation, pixel_height] let gdal_geotransform: GdalGeoTransform = [ values[2], // origin_x @@ -280,23 +286,36 @@ pub struct EoBand { /// Map a GDAL raster band channel index for a dataset band within an asset. /// /// If the asset has no `bands` metadata, returns channel 1 (single-band asset). -/// If the asset has bands, matches by `band_name` against asset band names. -/// If the asset has exactly one band and no `band_name` is required, returns -/// channel 1 (single-band asset treated the same as no band metadata). +/// If the asset has exactly one band, returns channel 1: a single-band asset has +/// only one GDAL raster band, and STAC servers commonly label it with a short +/// code (e.g. `B10`) while the configured band name may be the human-readable +/// asset title (e.g. `Thermal Infrared 10.9 (band 10) - 100m`), so a strict name +/// match would wrongly skip the asset. +/// If the asset has multiple bands, matches by `band_name` against asset band +/// names to select the channel. /// Returns `None` if the required band is not found. pub fn rasterband_channel_for_dataset_band( asset: &stac::Asset, required_band_name: Option<&str>, ) -> Option { - if asset.bands.is_empty() || (asset.bands.len() == 1 && required_band_name.is_none()) { - if required_band_name.is_some() && asset.bands.is_empty() { - tracing::warn!( - "STAC asset with href {} does not include bands, but dataset configuration requires a band name. Skipping asset.", - asset.href + if asset.bands.is_empty() { + // No `bands` metadata: assume a single-band raster and map to channel 1. + // STAC servers commonly omit `bands` for single-band products (e.g. + // Sentinel-2 SCL/CLD/SNW, Landsat QA bands), even when the mapping + // configures an explicit band name. Skipping here would silently lose + // the band, so proceed with channel 1 (the single-band path below does + // the same regardless of the requested name). + if required_band_name.is_some() { + tracing::debug!( + "STAC asset with href {} does not include bands, but dataset configuration requires band name {:?}. Assuming single-band raster (channel 1).", + asset.href, + required_band_name ); - return None; } + return Some(1); + } + if asset.bands.len() == 1 { return Some(1); } @@ -325,21 +344,38 @@ pub fn rasterband_channel_for_dataset_band( Some(asset_band_idx + 1) } +/// Parsed band information from a STAC 1.1.0 asset. +/// +/// Keeps the asset's display title separate from the individual band names so +/// callers can match an asset by its real STAC title and select the raster +/// channel by band name, without encoding the band name into the title (e.g. +/// `True color image [B02]`). +#[derive(Debug, Clone, PartialEq)] +pub struct AssetBandInfo { + pub asset_title: String, + pub band_names: Vec, +} + /// Derive band names from a STAC 1.1.0 `Asset`, using the `bands` field. -pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result, String> { +/// +/// For assets without `bands` metadata or with exactly one band, the single +/// band is named after the asset title. For multi-band assets the individual +/// STAC band names (e.g. `B04`) are returned, so the mapping can reference the +/// exact raster channel while keeping the real asset title. +pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result { let asset_title = asset .title .as_deref() - .ok_or_else(|| "Missing title in asset metadata".to_string())?; + .ok_or_else(|| "Missing title in asset metadata".to_string())? + .to_string(); let bands = &asset.bands; - if bands.is_empty() { - return Ok(vec![asset_title.to_string()]); - } - - if bands.len() == 1 { - return Ok(vec![asset_title.to_string()]); + if bands.is_empty() || bands.len() == 1 { + return Ok(AssetBandInfo { + asset_title: asset_title.clone(), + band_names: vec![asset_title], + }); } let mut names = Vec::new(); @@ -347,18 +383,24 @@ pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result, let Some(band_name) = &band.name else { return Err("Band is missing name for multi-band asset".to_string()); }; - names.push(format!("{asset_title} [{band_name}]")); + names.push(band_name.clone()); } - Ok(names) + Ok(AssetBandInfo { + asset_title, + band_names: names, + }) } /// Derive band names from a STAC 1.1.0 `ItemAsset`, using the `bands` additional field. -pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result, String> { +/// +/// See [`band_names_from_asset_v1_1_0`] for the naming semantics. +pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result { let asset_title = asset .title .as_deref() - .ok_or_else(|| "Missing title in asset metadata".to_string())?; + .ok_or_else(|| "Missing title in asset metadata".to_string())? + .to_string(); let band_names = asset .additional_fields @@ -366,15 +408,17 @@ pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result Result geoengine_operators::util::Result { + let bands = RasterBandDescriptors::new( + self.dataset + .bands + .iter() + .map(|b| b.band_descriptor.clone()) + .collect(), + )?; + Ok(RasterResultDescriptor { data_type: self.dataset.data_type, spatial_reference: self.dataset.projection.into(), @@ -264,7 +272,7 @@ impl dimension: self.time_dimension, }, spatial_grid: self.dataset.spatial_grid, - bands: RasterBandDescriptors::new_multiple_bands(self.dataset.bands.len() as u32), + bands, }) } @@ -507,13 +515,13 @@ impl StacMultiBandMetaData { common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); for (dataset_band_idx, dataset_band) in self.dataset.bands.iter().enumerate() { - if dataset_band.asset_title != asset_title { + if dataset_band.asset_band.asset_title != asset_title { continue; } let Some(rasterband_channel) = common::rasterband_channel_for_dataset_band( asset, - dataset_band.band_name.as_deref(), + dataset_band.asset_band.band_name.as_deref(), ) else { continue; }; @@ -822,7 +830,8 @@ mod tests { use geoengine_datatypes::util::Identifier; use geoengine_operators::engine::SpatialGridDescriptor; use geoengine_operators::engine::{ - MetaData, MetaDataProvider, RasterResultDescriptor, WorkflowOperatorPath, + MetaData, MetaDataProvider, RasterBandDescriptor, RasterResultDescriptor, + WorkflowOperatorPath, }; use geoengine_operators::source::{ MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, @@ -861,12 +870,18 @@ mod tests { ), bands: vec![ crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "NIR 1 (band 8) - 10m".to_owned(), - band_name: Some("B08".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "NIR 1 (band 8) - 10m".to_owned(), + band_name: Some("B08".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B08".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Red (band 4) - 10m".to_owned(), - band_name: Some("B04".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Red (band 4) - 10m".to_owned(), + band_name: Some("B04".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B04".to_owned()), }, ], }], @@ -1060,24 +1075,39 @@ mod tests { ), bands: vec![ crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Blue (band 2) - 10m".to_owned(), - band_name: Some("B02".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Blue (band 2) - 10m".to_owned(), + band_name: Some("B02".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B02".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Green (band 3) - 10m".to_owned(), - band_name: Some("B03".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Green (band 3) - 10m".to_owned(), + band_name: Some("B03".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B03".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Water vapour (WVP) - 10m".to_owned(), - band_name: Some("WVP".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Water vapour (WVP) - 10m".to_owned(), + band_name: Some("WVP".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("WVP".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "NIR 1 (band 8) - 10m".to_owned(), - band_name: Some("B08".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "NIR 1 (band 8) - 10m".to_owned(), + band_name: Some("B08".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B08".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Red (band 4) - 10m".to_owned(), - band_name: Some("B04".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Red (band 4) - 10m".to_owned(), + band_name: Some("B04".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B04".to_owned()), }, ], }, @@ -1097,12 +1127,18 @@ mod tests { ), bands: vec![ crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Aerosol optical thickness (AOT) - 20m".to_owned(), - band_name: Some("AOT".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Aerosol optical thickness (AOT) - 20m".to_owned(), + band_name: Some("AOT".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("AOT".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Scene classification map (SCL) - 20m".to_owned(), - band_name: Some("SCL".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Scene classification map (SCL) - 20m".to_owned(), + band_name: Some("SCL".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("SCL".to_owned()), }, ], }, diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index f4dbb8cd30..592cdc7847 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -8,7 +8,7 @@ use geoengine_datatypes::dataset::DataProviderId; use geoengine_datatypes::primitives::{SpatialResolution, TimeDimension}; use geoengine_datatypes::raster::RasterDataType; use geoengine_datatypes::spatial_reference::SpatialReference; -use geoengine_operators::engine::SpatialGridDescriptor; +use geoengine_operators::engine::{RasterBandDescriptor, SpatialGridDescriptor}; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -72,13 +72,62 @@ pub struct StacProviderDataset { pub bands: Vec, } +/// A band inside a STAC asset. +/// +/// *Addresses* the band in the asset files of a STAC collection: +/// [`asset_title`](StacAssetBand::asset_title) selects the asset file, +/// [`band_name`](StacAssetBand::band_name) selects the raster channel within +/// it. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSql, FromSql)] -#[postgres(name = "StacProviderDatasetBand")] -pub struct StacProviderDatasetBand { +#[postgres(name = "StacAssetBand")] +pub struct StacAssetBand { + /// The title of the STAC asset in the collection. Used to *address* the + /// asset file that contains this band (matched against the STAC asset + /// `title`). pub asset_title: String, + /// The name of the band *within* the asset file. + /// + /// Matches the STAC `bands[].name` metadata (e.g. `B04`) to select the + /// raster channel inside the asset. `None` for single-band assets. pub band_name: Option, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSql, FromSql)] +#[postgres(name = "StacProviderDatasetBand")] +pub struct StacProviderDatasetBand { + /// The band inside the STAC asset that this dataset band reads from. + /// + /// This is the *addressing* information: which asset file + /// ([`StacAssetBand::asset_title`]) and which raster channel within it + /// ([`StacAssetBand::band_name`]). + pub asset_band: StacAssetBand, + /// The band descriptor of the resulting geo engine dataset layer. + /// + /// This is independent of [`Self::asset_band`], which *addresses* the band + /// inside the asset files. During discovery it is populated with the same + /// naming fallback (`asset_band.band_name`, then + /// `asset_band.asset_title`) and a unitless measurement. + pub band_descriptor: RasterBandDescriptor, +} + +impl StacProviderDatasetBand { + /// Create a dataset band whose resulting result-descriptor band is + /// unitless and named after the asset band using the discovery fallback + /// ([`StacAssetBand::band_name`], else [`StacAssetBand::asset_title`]). + pub fn new_unitless(asset_band: StacAssetBand) -> Self { + let band_descriptor = RasterBandDescriptor::new_unitless( + asset_band + .band_name + .clone() + .unwrap_or_else(|| asset_band.asset_title.clone()), + ); + Self { + asset_band, + band_descriptor, + } + } +} + #[async_trait] impl DataProviderDefinition for StacDataProviderDefinition { async fn initialize(self: Box, _db: D) -> crate::error::Result> { diff --git a/geoengine/services/src/datasets/upload.rs b/geoengine/services/src/datasets/upload.rs index 64987ef57f..91de9971ec 100644 --- a/geoengine/services/src/datasets/upload.rs +++ b/geoengine/services/src/datasets/upload.rs @@ -56,6 +56,13 @@ impl<'de> Deserialize<'de> for VolumeName { impl AdjustFilePath for Volume { fn adjust_file_path(&self, file_path: &Path) -> Result { + if self.name.0 == "external" { + // external data file path must not be adjusted + // TODO: remove this once we have proper volume management + // TODO: ensure the file path actually points to external data + return Ok(file_path.to_path_buf()); + } + let _file_name = file_path.file_name().ok_or(error::Error::PathIsNotAFile)?; path_with_base_path(&self.path, file_path) diff --git a/geoengine/services/src/util/mod.rs b/geoengine/services/src/util/mod.rs index 7f841913c2..e24129559d 100644 --- a/geoengine/services/src/util/mod.rs +++ b/geoengine/services/src/util/mod.rs @@ -24,6 +24,7 @@ pub mod openapi_visitors; pub mod operators; pub mod parsing; pub mod postgres; +pub mod retry; pub mod sentinel_2_utm_zones; pub mod server; // TODO: refactor to be gated by `#[cfg(test)]` diff --git a/geoengine/services/src/util/retry.rs b/geoengine/services/src/util/retry.rs new file mode 100644 index 0000000000..a71d5c4407 --- /dev/null +++ b/geoengine/services/src/util/retry.rs @@ -0,0 +1,187 @@ +//! Generic HTTP retry helper with configurable stop/retry conditions. +//! +//! [`retry_http`] retries an async operation with exponential backoff until it +//! succeeds, the configured attempt limit is reached, or the error matches a +//! *terminal* condition. Terminal conditions can be declared by HTTP status +//! code and/or by substring in the error's display message, either as "stop +//! retrying on these" or as "only retry on these" (whitelist). + +use std::time::Duration; + +use tracing::{error, warn}; + +/// Default number of attempts before giving up. +pub const DEFAULT_MAX_RETRIES: u32 = 10; +/// Default initial backoff delay in milliseconds (doubles after each attempt). +pub const DEFAULT_INITIAL_DELAY_MS: u64 = 1000; + +/// Controls when [`retry_http`] keeps retrying and when it gives up. +/// +/// Two complementary mechanisms are available, each applicable to HTTP status +/// codes and to substrings of the error's `Display` message: +/// +/// - `stop_on_*`: errors matching these are terminal and returned immediately. +/// - `retry_on_*`: if non-empty, a whitelist — only matching errors are +/// retried, everything else is terminal. +/// +/// With no conditions set, every error is retried (plain exponential backoff). +#[derive(Debug, Clone)] +pub struct RetryPolicy { + pub max_retries: u32, + pub initial_delay_ms: u64, + /// HTTP status codes on which to stop retrying immediately. + pub stop_on_status: Vec, + /// HTTP status codes to retry (whitelist; empty = retry all non-terminal errors). + pub retry_on_status: Vec, + /// Error-message substrings on which to stop retrying immediately. + pub stop_on_message_contains: Vec, + /// Error-message substrings to retry (whitelist; empty = retry all non-terminal errors). + pub retry_on_message_contains: Vec, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_retries: DEFAULT_MAX_RETRIES, + initial_delay_ms: DEFAULT_INITIAL_DELAY_MS, + stop_on_status: Vec::new(), + retry_on_status: Vec::new(), + stop_on_message_contains: Vec::new(), + retry_on_message_contains: Vec::new(), + } + } +} + +impl RetryPolicy { + pub fn new() -> Self { + Self::default() + } + + #[must_use] + pub fn max_retries(mut self, max_retries: u32) -> Self { + self.max_retries = max_retries; + self + } + + #[must_use] + pub fn initial_delay_ms(mut self, initial_delay_ms: u64) -> Self { + self.initial_delay_ms = initial_delay_ms; + self + } + + /// Abort retrying on the given HTTP status codes (e.g. `[400, 404]`). + #[must_use] + pub fn stop_on_status(mut self, codes: &[u16]) -> Self { + self.stop_on_status.extend_from_slice(codes); + self + } + + /// Only retry on the given HTTP status codes; everything else is terminal. + #[must_use] + pub fn retry_on_status(mut self, codes: &[u16]) -> Self { + self.retry_on_status.extend_from_slice(codes); + self + } + + /// Abort retrying when the error message contains any of the given substrings. + #[must_use] + pub fn stop_on_message(mut self, substrings: &[&str]) -> Self { + self.stop_on_message_contains + .extend(substrings.iter().map(ToString::to_string)); + self + } + + /// Only retry when the error message contains any of the given substrings. + #[must_use] + pub fn retry_on_message(mut self, substrings: &[&str]) -> Self { + self.retry_on_message_contains + .extend(substrings.iter().map(ToString::to_string)); + self + } +} + +/// Retry an operation with exponential backoff, honouring `policy`. +/// +/// `extract_status` maps an error to an optional HTTP status code so the policy +/// can match `stop_on_status`/`retry_on_status`; the error's `Display` output is +/// used for the message-substring conditions. +pub async fn retry_http( + mut operation: F, + operation_name: &str, + policy: &RetryPolicy, + extract_status: impl Fn(&E) -> Option, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + E: std::fmt::Display, +{ + let mut attempt = 0; + loop { + match operation().await { + Ok(result) => return Ok(result), + Err(err) => { + if is_terminal(&err, policy, &extract_status) { + return Err(err); + } + + attempt += 1; + if attempt >= policy.max_retries { + error!( + "{operation_name} failed after {} attempts: {err}", + policy.max_retries + ); + return Err(err); + } + + let delay = Duration::from_millis(policy.initial_delay_ms * 2_u64.pow(attempt - 1)); + warn!( + "{operation_name} failed (attempt {attempt}/{}): {err}. Retrying in {delay:?}...", + policy.max_retries + ); + tokio::time::sleep(delay).await; + } + } + } +} + +/// Decide whether `err` should abort the retry loop under `policy`. +fn is_terminal( + err: &E, + policy: &RetryPolicy, + extract_status: &impl Fn(&E) -> Option, +) -> bool { + let status = extract_status(err); + let message = err.to_string(); + + // Explicit stop conditions always win. + if let Some(status) = status + && policy.stop_on_status.contains(&status) + { + return true; + } + if policy + .stop_on_message_contains + .iter() + .any(|needle| message.contains(needle.as_str())) + { + return true; + } + + // Whitelist conditions: if configured, retry only on matches. + let status_allows_retry = if policy.retry_on_status.is_empty() { + true + } else { + status.is_some_and(|s| policy.retry_on_status.contains(&s)) + }; + let message_allows_retry = if policy.retry_on_message_contains.is_empty() { + true + } else { + policy + .retry_on_message_contains + .iter() + .any(|needle| message.contains(needle.as_str())) + }; + + !(status_allows_retry && message_allows_retry) +} diff --git a/geoengine/test_data/provider_defs_api/stac_sentinel2.json b/geoengine/test_data/provider_defs_api/stac_sentinel2.json index 8835e3e080..eb058e257c 100644 --- a/geoengine/test_data/provider_defs_api/stac_sentinel2.json +++ b/geoengine/test_data/provider_defs_api/stac_sentinel2.json @@ -55,28 +55,76 @@ }, "bands": [ { - "assetTitle": "Aerosol optical thickness (AOT) - 10m", - "bandName": "AOT" + "assetBand": { + "assetTitle": "Aerosol optical thickness (AOT) - 10m", + "bandName": "AOT" + }, + "bandDescriptor": { + "name": "AOT", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Blue (band 2) - 10m", - "bandName": "B02" + "assetBand": { + "assetTitle": "Blue (band 2) - 10m", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Green (band 3) - 10m", - "bandName": "B03" + "assetBand": { + "assetTitle": "Green (band 3) - 10m", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 1 (band 8) - 10m", - "bandName": "B08" + "assetBand": { + "assetTitle": "NIR 1 (band 8) - 10m", + "bandName": "B08" + }, + "bandDescriptor": { + "name": "B08", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red (band 4) - 10m", - "bandName": "B04" + "assetBand": { + "assetTitle": "Red (band 4) - 10m", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Water vapour (WVP) - 10m", - "bandName": "WVP" + "assetBand": { + "assetTitle": "Water vapour (WVP) - 10m", + "bandName": "WVP" + }, + "bandDescriptor": { + "name": "WVP", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -114,52 +162,148 @@ }, "bands": [ { - "assetTitle": "Aerosol optical thickness (AOT) - 20m", - "bandName": "AOT" + "assetBand": { + "assetTitle": "Aerosol optical thickness (AOT) - 20m", + "bandName": "AOT" + }, + "bandDescriptor": { + "name": "AOT", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Blue (band 2) - 20m", - "bandName": "B02" + "assetBand": { + "assetTitle": "Blue (band 2) - 20m", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Coastal aerosol (band 1) - 20m", - "bandName": "B01" + "assetBand": { + "assetTitle": "Coastal aerosol (band 1) - 20m", + "bandName": "B01" + }, + "bandDescriptor": { + "name": "B01", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Green (band 3) - 20m", - "bandName": "B03" + "assetBand": { + "assetTitle": "Green (band 3) - 20m", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 2 (band 8A) - 20m", - "bandName": "B8A" + "assetBand": { + "assetTitle": "NIR 2 (band 8A) - 20m", + "bandName": "B8A" + }, + "bandDescriptor": { + "name": "B8A", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red (band 4) - 20m", - "bandName": "B04" + "assetBand": { + "assetTitle": "Red (band 4) - 20m", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 1 (band 5) - 20m", - "bandName": "B05" + "assetBand": { + "assetTitle": "Red edge 1 (band 5) - 20m", + "bandName": "B05" + }, + "bandDescriptor": { + "name": "B05", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 2 (band 6) - 20m", - "bandName": "B06" + "assetBand": { + "assetTitle": "Red edge 2 (band 6) - 20m", + "bandName": "B06" + }, + "bandDescriptor": { + "name": "B06", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 3 (band 7) - 20m", - "bandName": "B07" + "assetBand": { + "assetTitle": "Red edge 3 (band 7) - 20m", + "bandName": "B07" + }, + "bandDescriptor": { + "name": "B07", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 1 (band 11) - 20m", - "bandName": "B11" + "assetBand": { + "assetTitle": "SWIR 1 (band 11) - 20m", + "bandName": "B11" + }, + "bandDescriptor": { + "name": "B11", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 2 (band 12) - 20m", - "bandName": "B12" + "assetBand": { + "assetTitle": "SWIR 2 (band 12) - 20m", + "bandName": "B12" + }, + "bandDescriptor": { + "name": "B12", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Water vapour (WVP) - 20m", - "bandName": "WVP" + "assetBand": { + "assetTitle": "Water vapour (WVP) - 20m", + "bandName": "WVP" + }, + "bandDescriptor": { + "name": "WVP", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -197,56 +341,160 @@ }, "bands": [ { - "assetTitle": "Aerosol optical thickness (AOT) - 60m", - "bandName": "AOT" + "assetBand": { + "assetTitle": "Aerosol optical thickness (AOT) - 60m", + "bandName": "AOT" + }, + "bandDescriptor": { + "name": "AOT", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Blue (band 2) - 60m", - "bandName": "B02" + "assetBand": { + "assetTitle": "Blue (band 2) - 60m", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Coastal aerosol (band 1) - 60m", - "bandName": "B01" + "assetBand": { + "assetTitle": "Coastal aerosol (band 1) - 60m", + "bandName": "B01" + }, + "bandDescriptor": { + "name": "B01", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Green (band 3) - 60m", - "bandName": "B03" + "assetBand": { + "assetTitle": "Green (band 3) - 60m", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 2 (band 8A) - 60m", - "bandName": "B8A" + "assetBand": { + "assetTitle": "NIR 2 (band 8A) - 60m", + "bandName": "B8A" + }, + "bandDescriptor": { + "name": "B8A", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 3 (band 9) - 60m", - "bandName": "B09" + "assetBand": { + "assetTitle": "NIR 3 (band 9) - 60m", + "bandName": "B09" + }, + "bandDescriptor": { + "name": "B09", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red (band 4) - 60m", - "bandName": "B04" + "assetBand": { + "assetTitle": "Red (band 4) - 60m", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 1 (band 5) - 60m", - "bandName": "B05" + "assetBand": { + "assetTitle": "Red edge 1 (band 5) - 60m", + "bandName": "B05" + }, + "bandDescriptor": { + "name": "B05", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 2 (band 6) - 60m", - "bandName": "B06" + "assetBand": { + "assetTitle": "Red edge 2 (band 6) - 60m", + "bandName": "B06" + }, + "bandDescriptor": { + "name": "B06", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 3 (band 7) - 60m", - "bandName": "B07" + "assetBand": { + "assetTitle": "Red edge 3 (band 7) - 60m", + "bandName": "B07" + }, + "bandDescriptor": { + "name": "B07", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 1 (band 11) - 60m", - "bandName": "B11" + "assetBand": { + "assetTitle": "SWIR 1 (band 11) - 60m", + "bandName": "B11" + }, + "bandDescriptor": { + "name": "B11", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 2 (band 12) - 60m", - "bandName": "B12" + "assetBand": { + "assetTitle": "SWIR 2 (band 12) - 60m", + "bandName": "B12" + }, + "bandDescriptor": { + "name": "B12", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Water vapour (WVP) - 60m", - "bandName": "WVP" + "assetBand": { + "assetTitle": "Water vapour (WVP) - 60m", + "bandName": "WVP" + }, + "bandDescriptor": { + "name": "WVP", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -284,16 +532,40 @@ }, "bands": [ { - "assetTitle": "True color image", - "bandName": "B02" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B03" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B04" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -331,27 +603,75 @@ }, "bands": [ { - "assetTitle": "Cloud probability (CLD) - 20m", - "bandName": "CLD" + "assetBand": { + "assetTitle": "Cloud probability (CLD) - 20m", + "bandName": "CLD" + }, + "bandDescriptor": { + "name": "CLD", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Scene classification map (SCL) - 20m" + "assetBand": { + "assetTitle": "Scene classification map (SCL) - 20m" + }, + "bandDescriptor": { + "name": "Scene classification map (SCL) - 20m", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Snow probability (SNW) - 20m", - "bandName": "SNW" + "assetBand": { + "assetTitle": "Snow probability (SNW) - 20m", + "bandName": "SNW" + }, + "bandDescriptor": { + "name": "SNW", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B02" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B03" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B04" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -389,27 +709,75 @@ }, "bands": [ { - "assetTitle": "Cloud probability (CLD) - 60m", - "bandName": "CLD" + "assetBand": { + "assetTitle": "Cloud probability (CLD) - 60m", + "bandName": "CLD" + }, + "bandDescriptor": { + "name": "CLD", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Scene classification map (SCL) - 60m" + "assetBand": { + "assetTitle": "Scene classification map (SCL) - 60m" + }, + "bandDescriptor": { + "name": "Scene classification map (SCL) - 60m", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Snow probability (SNW) - 60m", - "bandName": "SNW" + "assetBand": { + "assetTitle": "Snow probability (SNW) - 60m", + "bandName": "SNW" + }, + "bandDescriptor": { + "name": "SNW", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B02" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B03" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B04" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } } ] } diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json index a7a0fe3296..daa85db195 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-code-de.json +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -10,61 +10,155 @@ "timeDimension": { "type": "regular", "origin": 0, - "step": { "granularity": "days", "step": 1 } + "step": { + "granularity": "days", + "step": 1 + } }, "datasets": [ { "name": "sentinel-2-l2a EPSG:32632 U16 10m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", "dataType": "U16", - "resolution": { "x": 10.0, "y": 10.0 }, + "resolution": { + "x": 10.0, + "y": 10.0 + }, "projection": "EPSG:32632", "spatialGrid": { "spatialGrid": { "geoTransform": { - "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "originCoordinate": { + "x": 399960.0, + "y": 5700000.0 + }, "xPixelSize": 10.0, "yPixelSize": -10.0 }, "gridBounds": { - "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, - "bottomRightIdx": { "yIdx": 10979, "xIdx": 10979 } + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 10979, + "xIdx": 10979 + } } }, "descriptor": "source" }, "bands": [ - { "assetTitle": "Blue (band 2) - 10m", "bandName": null }, - { "assetTitle": "Green (band 3) - 10m", "bandName": null }, - { "assetTitle": "NIR 1 (band 8) - 10m", "bandName": null }, - { "assetTitle": "Red (band 4) - 10m", "bandName": null } + { + "assetBand": { + "assetTitle": "Blue (band 2) - 10m", + "bandName": "Blue (band 2) - 10m" + }, + "bandDescriptor": { + "name": "Blue (band 2) - 10m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Green (band 3) - 10m", + "bandName": "Green (band 3) - 10m" + }, + "bandDescriptor": { + "name": "Green (band 3) - 10m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "NIR 1 (band 8) - 10m", + "bandName": "NIR 1 (band 8) - 10m" + }, + "bandDescriptor": { + "name": "NIR 1 (band 8) - 10m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Red (band 4) - 10m", + "bandName": "Red (band 4) - 10m" + }, + "bandDescriptor": { + "name": "Red (band 4) - 10m", + "measurement": { + "type": "unitless" + } + } + } ] }, { "name": "sentinel-2-l2a EPSG:32632 U16 20m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", "dataType": "U16", - "resolution": { "x": 20.0, "y": 20.0 }, + "resolution": { + "x": 20.0, + "y": 20.0 + }, "projection": "EPSG:32632", "spatialGrid": { "spatialGrid": { "geoTransform": { - "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "originCoordinate": { + "x": 399960.0, + "y": 5700000.0 + }, "xPixelSize": 20.0, "yPixelSize": -20.0 }, "gridBounds": { - "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, - "bottomRightIdx": { "yIdx": 5489, "xIdx": 5489 } + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 5489, + "xIdx": 5489 + } } }, "descriptor": "source" }, "bands": [ - { "assetTitle": "SWIR 1 (band 11) - 20m", "bandName": null }, - { "assetTitle": "SWIR 2 (band 12) - 20m", "bandName": null } + { + "assetBand": { + "assetTitle": "SWIR 1 (band 11) - 20m", + "bandName": "SWIR 1 (band 11) - 20m" + }, + "bandDescriptor": { + "name": "SWIR 1 (band 11) - 20m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "SWIR 2 (band 12) - 20m", + "bandName": "SWIR 2 (band 12) - 20m" + }, + "bandDescriptor": { + "name": "SWIR 2 (band 12) - 20m", + "measurement": { + "type": "unitless" + } + } + } ] } ], - "pageLimit": 100 + "pageLimit": 100, + "queryTimeoutSecs": 60 } diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json index 08d2f1a7ef..541b6d794d 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -10,36 +10,96 @@ "timeDimension": { "type": "regular", "origin": 0, - "step": { "granularity": "days", "step": 1 } + "step": { + "granularity": "days", + "step": 1 + } }, "datasets": [ { "name": "landsat-c2-l1 EPSG:32632 U16 30m", "description": "Auto-discovered from STAC collection 'landsat-c2-l1'", "dataType": "U16", - "resolution": { "x": 30.0, "y": 30.0 }, + "resolution": { + "x": 30.0, + "y": 30.0 + }, "projection": "EPSG:32632", "spatialGrid": { "spatialGrid": { "geoTransform": { - "originCoordinate": { "x": 399960.0, "y": 5800020.0 }, + "originCoordinate": { + "x": 399960.0, + "y": 5800020.0 + }, "xPixelSize": 30.0, "yPixelSize": -30.0 }, "gridBounds": { - "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, - "bottomRightIdx": { "yIdx": 7999, "xIdx": 7999 } + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 7999, + "xIdx": 7999 + } } }, "descriptor": "source" }, "bands": [ - { "assetTitle": "Blue (band 2) - 30m", "bandName": null }, - { "assetTitle": "Green (band 3) - 30m", "bandName": null }, - { "assetTitle": "NIR (band 5) - 30m", "bandName": null }, - { "assetTitle": "Red (band 4) - 30m", "bandName": null } + { + "assetBand": { + "assetTitle": "Blue (band 2) - 30m", + "bandName": "Blue (band 2) - 30m" + }, + "bandDescriptor": { + "name": "Blue (band 2) - 30m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Green (band 3) - 30m", + "bandName": "Green (band 3) - 30m" + }, + "bandDescriptor": { + "name": "Green (band 3) - 30m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "NIR (band 5) - 30m", + "bandName": "NIR (band 5) - 30m" + }, + "bandDescriptor": { + "name": "NIR (band 5) - 30m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Red (band 4) - 30m", + "bandName": "Red (band 4) - 30m" + }, + "bandDescriptor": { + "name": "Red (band 4) - 30m", + "measurement": { + "type": "unitless" + } + } + } ] } ], - "pageLimit": 100 + "pageLimit": 100, + "queryTimeoutSecs": 60 } diff --git a/openapi.json b/openapi.json index 92a5df68da..582359b21c 100644 --- a/openapi.json +++ b/openapi.json @@ -12231,6 +12231,23 @@ } } }, + "StacAssetBand": { + "type": "object", + "required": [ + "assetTitle" + ], + "properties": { + "assetTitle": { + "type": "string" + }, + "bandName": { + "type": [ + "string", + "null" + ] + } + } + }, "StacDataProviderDefinition": { "type": "object", "required": [ @@ -12241,7 +12258,8 @@ "apiUrl", "collectionName", "timeDimension", - "datasets" + "datasets", + "pageLimit" ], "properties": { "type": { @@ -12291,6 +12309,10 @@ "$ref": "#/components/schemas/StacProviderDataset" } }, + "pageLimit": { + "type": "integer", + "format": "int64" + }, "queryTimeoutSecs": { "type": "integer", "format": "int64", @@ -12339,17 +12361,17 @@ "StacProviderDatasetBand": { "type": "object", "required": [ - "assetTitle" + "assetBand", + "bandDescriptor" ], "properties": { - "assetTitle": { - "type": "string" + "assetBand": { + "$ref": "#/components/schemas/StacAssetBand", + "description": "The band inside the STAC asset that this dataset band reads from\n(addressing: which asset file + which raster channel within it)." }, - "bandName": { - "type": [ - "string", - "null" - ] + "bandDescriptor": { + "$ref": "#/components/schemas/RasterBandDescriptor", + "description": "The band descriptor of the resulting geo engine dataset layer.\n\nIndependent of `assetBand`, which *addresses* the band inside the asset\nfiles. Populated by discovery with the naming fallback\n(`assetBand.bandName`, then `assetBand.assetTitle`) and a unitless\nmeasurement." } } }, From bb5067e8cc0cbef7f1bbbd7667048b1fab88648b Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 19:35:13 +0000 Subject: [PATCH 7/9] refactor migration --- geoengine/services/src/api/model/services.rs | 7 ++++++- .../src/contexts/migrations/current_schema.sql | 4 ++-- .../migrations/migration_0028_stac_provider.sql | 1 - .../migration_0029_stac_provider_band_name.rs | 7 ++++--- .../migration_0029_stac_provider_band_name.sql | 13 +++++++++++-- .../services/src/datasets/external/stac/mod.rs | 2 +- openapi.json | 8 ++++---- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 9b4f19a240..479f098237 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -1001,6 +1001,11 @@ pub struct StacProviderDatasetBand { /// (addressing: which asset file + which raster channel within it). pub asset_band: StacAssetBand, /// The band descriptor of the resulting geo engine dataset layer. + /// + /// Independent of `assetBand`, which *addresses* the band inside the asset + /// files. Populated by discovery with the naming fallback + /// (`assetBand.bandName`, then `assetBand.assetTitle`) and a unitless + /// measurement. pub band_descriptor: crate::api::model::operators::RasterBandDescriptor, } @@ -1150,10 +1155,10 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, pub datasets: Vec, - pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, + pub page_limit: i64, } fn default_query_timeout() -> i64 { diff --git a/geoengine/services/src/contexts/migrations/current_schema.sql b/geoengine/services/src/contexts/migrations/current_schema.sql index 4f3f26d53e..2834f97c05 100644 --- a/geoengine/services/src/contexts/migrations/current_schema.sql +++ b/geoengine/services/src/contexts/migrations/current_schema.sql @@ -912,8 +912,8 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], - page_limit bigint, - query_timeout_secs bigint + query_timeout_secs bigint, + page_limit bigint ); CREATE TYPE "DataProviderDefinition" AS ( diff --git a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql index e68600336f..cd9c408c63 100644 --- a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql +++ b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql @@ -29,7 +29,6 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], - page_limit bigint, query_timeout_secs bigint ); diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs index 330a23f7d6..5479390993 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs @@ -6,10 +6,11 @@ use async_trait::async_trait; use tokio_postgres::Transaction; /// This migration bundles the band addressing fields of `StacProviderDatasetBand` -/// into a nested `StacAssetBand` type and adds a `name` attribute of type -/// `RasterBandDescriptor` for the band in the resulting geo engine dataset +/// into a nested `StacAssetBand` type and adds a `band_descriptor` attribute of +/// type `RasterBandDescriptor` for the band in the resulting geo engine dataset /// layer, independent of `asset_title`/`band_name`, which address the band -/// inside the STAC asset files. +/// inside the STAC asset files. It also takes over the `page_limit` attribute +/// of `StacDataProviderDefinition` from the released migration 0028. pub struct Migration0029StacProviderBandName; #[async_trait] diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql index 55dde1e39a..a9113bcd18 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -12,6 +12,10 @@ -- `RasterBandDescriptor` of the geo engine dataset layer band, populated with -- the naming fallback ("use band_name, then asset_title") and a unitless -- measurement. +-- +-- Additionally, this migration takes over the `page_limit` attribute of +-- `StacDataProviderDefinition` that previously lived in the released migration +-- `0028_stac_provider`, so that migration stays untouched. CREATE TYPE "StacAssetBand" AS ( asset_title text, @@ -21,6 +25,11 @@ CREATE TYPE "StacAssetBand" AS ( ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE asset_band "StacAssetBand"; ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE band_descriptor "RasterBandDescriptor"; +-- `page_limit` was moved here from migration 0028, which is already released. +-- It must be added before the data migration below, since +-- `pg_temp.stac_migrate_provider_def` reads `(def).page_limit`. +ALTER TYPE "StacDataProviderDefinition" ADD ATTRIBUTE page_limit bigint; + -- Migrate existing stored STAC provider definitions: bundle the flat -- `asset_title`/`band_name` attributes into the new `asset_band` attribute. CREATE FUNCTION pg_temp.stac_migrate_bands( @@ -79,8 +88,8 @@ BEGIN (def).s3_config, (def).time_dimension, new_datasets, - (def).page_limit, - (def).query_timeout_secs + (def).query_timeout_secs, + (def).page_limit )::"StacDataProviderDefinition"; END; $$ LANGUAGE plpgsql; diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index 592cdc7847..d6b4cce9ed 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -33,10 +33,10 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, // TODO: should this be on dataset level? pub datasets: Vec, - pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, + pub page_limit: i64, } fn default_query_timeout() -> i64 { diff --git a/openapi.json b/openapi.json index 582359b21c..04b2f4db92 100644 --- a/openapi.json +++ b/openapi.json @@ -12309,14 +12309,14 @@ "$ref": "#/components/schemas/StacProviderDataset" } }, - "pageLimit": { - "type": "integer", - "format": "int64" - }, "queryTimeoutSecs": { "type": "integer", "format": "int64", "description": "Timeout in seconds for outgoing STAC API HTTP requests." + }, + "pageLimit": { + "type": "integer", + "format": "int64" } } }, From 07620cb478de5bab83fd3b7ef12a232a836e990a Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 20:36:46 +0000 Subject: [PATCH 8/9] adjust data path --- .../src/cli/stac_harvester/harvest.rs | 31 +-- .../src/datasets/external/stac/common.rs | 55 +--- .../datasets/external/stac/loading_info.rs | 247 +++--------------- 3 files changed, 60 insertions(+), 273 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 06c532fe89..9a0e2fc1ce 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, io::Read, str::FromStr, time::Instant}; +use std::{collections::HashMap, io::Read, path::PathBuf, str::FromStr, time::Instant}; use anyhow::Context; use chrono::Timelike; @@ -962,11 +962,7 @@ fn build_stac_query_params(params: &StacHarvest, page_limit: usize) -> Vec<(Stri query_params.push(("limit".to_string(), page_limit.to_string())); if params.filter_item_fields { - query_params.push(( - "fields".to_string(), - "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" - .to_string(), - )); + query_params.push(("fields".to_string(), common::STAC_ITEM_FIELDS.to_string())); } query_params @@ -1172,7 +1168,14 @@ fn try_create_tile_for_band( let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - let file_path = common::gdal_file_path(&asset.href)?; + let file_path = if asset.href.starts_with("http://") + || asset.href.starts_with("https://") + || asset.href.starts_with("s3://") + { + PathBuf::from(&asset.href) + } else { + return None; + }; let gdal_config_options = common::gdal_config_options_for_file_path(&file_path, provider_def.s3_config.as_ref()); @@ -1307,11 +1310,8 @@ mod tests { "tile dimensions should be positive" ); assert!( - tile.params - .file_path - .to_string_lossy() - .starts_with("/vsis3/"), - "file path should be a VSI path: {}", + tile.params.file_path.to_string_lossy().starts_with("s3://"), + "file path should be an S3 URL: {}", tile.params.file_path.display() ); } @@ -1397,11 +1397,8 @@ mod tests { "tile dimensions should be positive" ); assert!( - tile.params - .file_path - .to_string_lossy() - .starts_with("/vsis3/"), - "file path should be a VSI path: {}", + tile.params.file_path.to_string_lossy().starts_with("s3://"), + "file path should be an S3 URL: {}", tile.params.file_path.display() ); } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index e537bf6244..ecc2c1cfd9 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -13,10 +13,13 @@ use geoengine_datatypes::{ spatial_reference::SpatialReference, }; use serde::Deserialize; -use std::path::PathBuf; use super::StacProviderS3Config; +/// STAC `fields` query parameter used to keep item responses small while including all +/// metadata needed by the provider (loading info) and the harvester (discovery/mapping). +pub const STAC_ITEM_FIELDS: &str = "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform"; + // --------------------------------------------------------------------------- // STAC extension version types // --------------------------------------------------------------------------- @@ -255,23 +258,6 @@ pub fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option Option { - if href.starts_with("http") { - return Some(PathBuf::from(format!("/vsicurl/{href}"))); - } - - href.strip_prefix("s3://") - .map(|s3_path| PathBuf::from(format!("/vsis3/{s3_path}"))) -} - // --------------------------------------------------------------------------- // Band processing helpers // --------------------------------------------------------------------------- @@ -558,7 +544,7 @@ pub fn is_jp2_media_type(media_type: Option<&str>) -> bool { // GDAL config options // --------------------------------------------------------------------------- -/// Build GDAL configuration options for `/vsis3/` paths. +/// Build GDAL configuration options for S3-backed file paths. /// /// Returns the common options plus S3-specific credentials when an S3 config is provided. pub fn gdal_config_options_for_s3( @@ -583,16 +569,16 @@ pub fn gdal_config_options_for_s3( options } -/// Build GDAL configuration options for a VSI file path, including common CURL/S3 options. +/// Build GDAL configuration options for a remote (HTTP or S3) file path, including common CURL/S3 options. pub fn gdal_config_options_for_file_path( file_path: &std::path::Path, s3_config: Option<&StacProviderS3Config>, ) -> Option> { let file_path_str = file_path.to_string_lossy(); - let is_vsi_s3 = file_path_str.starts_with("/vsis3/"); - let is_vsi_curl = file_path_str.starts_with("/vsicurl/"); + let is_s3 = file_path_str.starts_with("s3://"); + let is_http = file_path_str.starts_with("http://") || file_path_str.starts_with("https://"); - if !is_vsi_s3 && !is_vsi_curl { + if !is_s3 && !is_http { return None; } @@ -607,7 +593,7 @@ pub fn gdal_config_options_for_file_path( ), ]; - if is_vsi_s3 { + if is_s3 { options.extend(gdal_config_options_for_s3(s3_config)); } @@ -753,27 +739,6 @@ mod tests { assert_eq!(raster_data_type_from_stac_data_type_str("unknown"), None); } - // ----------------------------------------------------------------------- - // gdal_file_path - // ----------------------------------------------------------------------- - - #[test] - fn test_gdal_file_path_http() { - let path = gdal_file_path("https://example.com/file.tif").expect("should parse"); - assert_eq!(path, PathBuf::from("/vsicurl/https://example.com/file.tif")); - } - - #[test] - fn test_gdal_file_path_s3() { - let path = gdal_file_path("s3://bucket/key/file.tif").expect("should parse"); - assert_eq!(path, PathBuf::from("/vsis3/bucket/key/file.tif")); - } - - #[test] - fn test_gdal_file_path_unsupported() { - assert!(gdal_file_path("/local/path.tif").is_none()); - } - // ----------------------------------------------------------------------- // parse_epsg_from_proj_code // ----------------------------------------------------------------------- diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 60ff02c250..b82b85d780 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -25,6 +25,7 @@ use geoengine_operators::source::{ OgrSourceDataset, TileFile, }; use stac::Item; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use tracing::debug; @@ -356,36 +357,33 @@ impl StacMultiBandMetaData { let time_end = time_interval.end(); let query_params = vec![ - ( - "bbox".to_owned(), - format!( - "{},{},{},{}", - bbox.lower_left().x, - bbox.lower_left().y, - bbox.upper_right().x, - bbox.upper_right().y - ), - ), - ( - "datetime".to_owned(), - format!( - "{}/{}", - time_start - .as_date_time() - .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? - .to_datetime_string_with_millis(), - time_end - .as_date_time() - .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? - .to_datetime_string_with_millis(), - ), - ), - ("limit".to_owned(), self.page_limit.to_string()), - ( - "fields".to_owned(), - "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform".to_owned(), - ), - ]; + ( + "bbox".to_owned(), + format!( + "{},{},{},{}", + bbox.lower_left().x, + bbox.lower_left().y, + bbox.upper_right().x, + bbox.upper_right().y + ), + ), + ( + "datetime".to_owned(), + format!( + "{}/{}", + time_start + .as_date_time() + .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? + .to_datetime_string_with_millis(), + time_end + .as_date_time() + .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? + .to_datetime_string_with_millis(), + ), + ), + ("limit".to_owned(), self.page_limit.to_string()), + ("fields".to_owned(), common::STAC_ITEM_FIELDS.to_owned()), + ]; Ok(query_params) } @@ -508,8 +506,14 @@ impl StacMultiBandMetaData { .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - let file_path = common::gdal_file_path(&asset.href) - .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)?; + let file_path = if asset.href.starts_with("http://") + || asset.href.starts_with("https://") + || asset.href.starts_with("s3://") + { + PathBuf::from(&asset.href) + } else { + return Err(geoengine_operators::error::Error::InvalidDataProviderConfig.into()); + }; let gdal_config_options = common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); @@ -554,87 +558,6 @@ impl StacMultiBandMetaData { Ok(()) } - - fn rasterband_channel_for_dataset_band( - asset: &stac::Asset, - required_band_name: Option<&str>, - ) -> Option { - if asset.bands.is_empty() { - if required_band_name.is_some() { - tracing::warn!( - "STAC asset with href {} does not include bands, but dataset configuration requires a band name. Skipping asset.", - asset.href - ); - return None; - } - - return Some(1); - } - - let Some(required_band_name) = required_band_name else { - tracing::warn!( - "STAC asset with href {} includes bands, but dataset configuration does not specify a band name. Skipping asset.", - asset.href - ); - return None; - }; - - let Some(asset_band_idx) = asset - .bands - .iter() - .position(|asset_band| asset_band.name.as_deref() == Some(required_band_name)) - else { - tracing::debug!( - "Skipping asset with href {} due to missing required band {}", - asset.href, - required_band_name - ); - return None; - }; - - Some(asset_band_idx + 1) - } - - fn gdal_config_options_for_file_path(&self, file_path: &Path) -> Option> { - let file_path_str = file_path.to_string_lossy(); - let is_vsi_s3 = file_path_str.starts_with("s3://"); - let is_vsi_curl = - file_path_str.starts_with("http://") || file_path_str.starts_with("https://"); - - if !is_vsi_s3 && !is_vsi_curl { - return None; - } - - let mut options = vec![ - ( - "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), - "EMPTY_DIR".to_owned(), - ), - ( - "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), - ".tif,.tiff,.jp2".to_owned(), - ), - ]; - - if !is_vsi_s3 { - return Some(options); - } - - if let Some(config) = self.s3_config.as_ref() { - options.push(("AWS_S3_ENDPOINT".to_owned(), config.endpoint.clone())); - options.push(("AWS_VIRTUAL_HOSTING".to_owned(), "FALSE".to_owned())); // TODO: make configurable? - - if let Some(access_key) = &config.access_key { - options.push(("AWS_ACCESS_KEY_ID".to_owned(), access_key.clone())); - } - - if let Some(secret_key) = &config.secret_key { - options.push(("AWS_SECRET_ACCESS_KEY".to_owned(), secret_key.clone())); - } - } - - Some(options) - } } fn stac_query_bbox( @@ -679,104 +602,6 @@ fn stac_query_time_interval( } } -fn gdal_file_path(href: &str) -> Option { - if href.starts_with("http://") || href.starts_with("https://") || href.starts_with("s3://") { - return Some(PathBuf::from(href)); - } - - None -} - -fn proj_shape_from_fields(fields: &serde_json::Map) -> Option<(usize, usize)> { - let proj_shape = fields.get("proj:shape")?.as_array()?; - if proj_shape.len() != 2 { - return None; - } - - let height = proj_shape.first()?.as_u64()? as usize; - let width = proj_shape.get(1)?.as_u64()? as usize; - - Some((height, width)) -} - -fn geo_transform_from_fields(fields: &serde_json::Map) -> Option { - let proj_transform = fields.get("proj:transform")?; - let proj_transform_array = proj_transform.as_array()?; - if proj_transform_array.len() != 6 { - return None; - } - - let proj_transform_values = proj_transform_array - .iter() - .map(Value::as_f64) - .collect::>>()?; - - let gdal_geotransform = [ - proj_transform_values[2], - proj_transform_values[0], - proj_transform_values[1], - proj_transform_values[5], - proj_transform_values[3], - proj_transform_values[4], - ]; - - Some(gdal_geotransform.into()) -} - -fn data_type_from_asset_v1_1_0(asset: &stac::Asset) -> Option { - asset - .data_type - .as_ref() - .and_then(raster_data_type_from_stac_data_type) -} - -fn raster_data_type_from_stac_data_type( - data_type: &stac_extensions::raster::DataType, -) -> Option { - match data_type { - stac_extensions::raster::DataType::UInt8 => Some(RasterDataType::U8), - stac_extensions::raster::DataType::UInt16 => Some(RasterDataType::U16), - stac_extensions::raster::DataType::UInt32 => Some(RasterDataType::U32), - stac_extensions::raster::DataType::Int16 => Some(RasterDataType::I16), - stac_extensions::raster::DataType::Int32 => Some(RasterDataType::I32), - stac_extensions::raster::DataType::Float32 => Some(RasterDataType::F32), - stac_extensions::raster::DataType::Float64 => Some(RasterDataType::F64), - _ => None, - } -} - -fn proj_code_matches_dataset( - fields: &serde_json::Map, - dataset_projection: SpatialReference, -) -> bool { - let Some(code) = fields.get("proj:code") else { - return false; - }; - - let Some(proj_code) = proj_code_as_srs_string(code) else { - return false; - }; - - proj_code == dataset_projection.to_string() -} - -fn proj_code_as_srs_string(value: &Value) -> Option { - if let Some(code_number) = value.as_u64() { - return Some(format!("EPSG:{code_number}")); - } - - let code_str = value.as_str()?.trim(); - if code_str.contains(':') { - return Some(code_str.to_ascii_uppercase()); - } - - if let Ok(code_number) = code_str.parse::() { - return Some(format!("EPSG:{code_number}")); - } - - None -} - #[async_trait] impl MetaDataProvider< From 11c49462b1d70688808d81f82fb6cd7639a47c3f Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 20:52:43 +0000 Subject: [PATCH 9/9] lint --- .../migration_0029_stac_provider_band_name.sql | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql index a9113bcd18..c45e23ebd2 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -95,10 +95,13 @@ END; $$ LANGUAGE plpgsql; UPDATE layer_providers -SET definition = ROW( - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - pg_temp.stac_migrate_provider_def((definition).stac_data_provider_definition) -)::"DataProviderDefinition" +SET + definition = ROW( + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pg_temp.stac_migrate_provider_def( + (definition).stac_data_provider_definition + ) + )::"DataProviderDefinition" WHERE (definition).stac_data_provider_definition IS NOT NULL; -- Drop the now-redundant flat addressing attributes.