diff --git a/crates/apollo_deployments/src/deployment_definitions_test.rs b/crates/apollo_deployments/src/deployment_definitions_test.rs index dfe83d386ba..db2cff2c349 100644 --- a/crates/apollo_deployments/src/deployment_definitions_test.rs +++ b/crates/apollo_deployments/src/deployment_definitions_test.rs @@ -2,15 +2,17 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::env; use std::fs::File; +use apollo_config::{FIELD_SEPARATOR, IS_NONE_MARK}; use apollo_infra_utils::dumping::serialize_to_file; use apollo_infra_utils::path::resolve_project_relative_path; use apollo_node_config::config_utils::private_parameters; +use apollo_node_config::node_config::CONFIG_SCHEMA_PATH; use serde_json::{to_value, Map, Value}; use strum::IntoEnumIterator; use tempfile::NamedTempFile; use crate::deployment_definitions::ComponentConfigInService; -use crate::service::NodeType; +use crate::service::{NodeService, NodeType}; use crate::test_utils::SecretsConfigOverride; const SECRETS_FOR_TESTING_ENV_PATH: &str = @@ -58,25 +60,14 @@ fn duplicate_config_entries() { for node_type in NodeType::iter() { for node_service in node_type.all_service_names() { - let deployment_file_path = node_service.replacer_deployment_file_path(); - let deployment_file = File::open(deployment_file_path).unwrap(); - - let mut application_config_files: Vec = - serde_json::from_reader(deployment_file) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) - .unwrap(); + let mut config_files_to_load = application_config_files(&node_service); // Add the secrets config file path to the config load command. - application_config_files.push(secrets_file_path.to_string()); + config_files_to_load.push(secrets_file_path.to_string()); let mut key_to_files: BTreeMap> = BTreeMap::new(); - for application_config_file in &application_config_files { - let file = File::open(application_config_file).unwrap(); - let json_map: Map = serde_json::from_reader(file) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) - .unwrap(); - - for key in json_map.keys() { + for application_config_file in &config_files_to_load { + for key in config_file_params(application_config_file).keys() { key_to_files .entry(key.clone()) .or_default() @@ -99,6 +90,72 @@ fn duplicate_config_entries() { } } +/// Test that every config param a service needs is set explicitly in its application config +/// files. A deployed pod loads them through `SequencerNodeConfig::load_and_process`, which passes +/// `ignore_default_values = true`, so an absent param either fails deserialization at startup or +/// silently takes a value no environment can override. +#[test] +fn all_config_params_are_set_in_service_configs() { + env::set_current_dir(resolve_project_relative_path("").unwrap()) + .expect("Couldn't set working dir."); + + let config_schema: Map = serde_json::from_reader( + File::open(resolve_project_relative_path(CONFIG_SCHEMA_PATH).unwrap()).unwrap(), + ) + .unwrap(); + + // Params carrying a `pointer_target` instead of a `value` are resolved from the param they + // point at, and private params come from the secrets file; neither belongs in an app config. + let private_params = private_parameters(); + let required_params: BTreeSet<&String> = config_schema + .iter() + .filter(|(param_path, param)| { + param.get("value").is_some() && !private_params.contains(*param_path) + }) + .map(|(param_path, _)| param_path) + .collect(); + + let is_none_suffix = format!("{FIELD_SEPARATOR}{IS_NONE_MARK}"); + + for node_type in NodeType::iter() { + for node_service in node_type.all_service_names() { + let mut set_params: Map = Map::new(); + for application_config_file in &application_config_files(&node_service) { + set_params.extend(config_file_params(application_config_file)); + } + + // A `None` optional keeps only its own `#is_none` mark: `update_optional_values` drops + // both the params below it and its own param. + let disabled_params: BTreeSet = set_params + .iter() + .filter(|(param_path, value)| { + param_path.ends_with(&is_none_suffix) && *value == &Value::Bool(true) + }) + .map(|(param_path, _)| param_path.strip_suffix(&is_none_suffix).unwrap().to_owned()) + .collect(); + + let missing_params: Vec<&String> = required_params + .iter() + .copied() + .filter(|param_path| { + !set_params.contains_key(*param_path) + && !disabled_params.iter().any(|disabled| { + *param_path == disabled + || param_path.starts_with(&format!("{disabled}{FIELD_SEPARATOR}")) + }) + }) + .collect(); + + assert!( + missing_params.is_empty(), + "Service {node_service:?} of node type {node_type} does not set \ + {missing_params:#?}. Add each param to the matching file under \ + crates/apollo_deployments/resources/app_configs and regenerate." + ); + } + } +} + /// Test that the private values in the apollo node config schema match the secrets config override /// schema. #[test] @@ -180,3 +237,13 @@ fn l1_components_state_consistency() { ); } } + +/// The application config files a service's pod mounts, in the order the deployment lists them. +fn application_config_files(node_service: &NodeService) -> Vec { + let deployment_file = File::open(node_service.replacer_deployment_file_path()).unwrap(); + serde_json::from_reader(deployment_file).unwrap() +} + +fn config_file_params(application_config_file: &str) -> Map { + serde_json::from_reader(File::open(application_config_file).unwrap()).unwrap() +}