From d73b85db8a28644a252c7215b24ef5fd08f80f7f Mon Sep 17 00:00:00 2001 From: BryanFRD Date: Sat, 10 Oct 2026 17:06:43 +0200 Subject: [PATCH] refactor(config): split config.rs into storage, auth and dashboard modules --- server/src/config.rs | 496 +-------------------------------- server/src/config/auth.rs | 238 ++++++++++++++++ server/src/config/dashboard.rs | 55 ++++ server/src/config/forges.rs | 2 +- server/src/config/storage.rs | 201 +++++++++++++ server/src/config/tests.rs | 9 + 6 files changed, 514 insertions(+), 487 deletions(-) create mode 100644 server/src/config/auth.rs create mode 100644 server/src/config/dashboard.rs create mode 100644 server/src/config/storage.rs diff --git a/server/src/config.rs b/server/src/config.rs index 5a9c2b7..c006077 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -4,13 +4,21 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::time::Duration; -use crate::auth::Namespaces; use crate::model::Action; use crate::namespace::Namespace; +mod auth; +mod dashboard; mod forges; +mod storage; -pub use forges::Forge; +pub use self::auth::{Auth, GithubApp, Provider}; +pub use self::dashboard::Dashboard; +pub use self::forges::Forge; +pub use self::storage::{AzureCredential, Dialect, DiskCache, GcsCredential, KeySource, Storage}; + +use self::dashboard::dashboard; +use self::storage::encryption_key; #[derive(Debug, Clone)] pub struct Config { @@ -44,335 +52,6 @@ pub struct Config { pub forges: Vec, } -#[derive(Debug, Clone)] -pub struct Dashboard { - pub dir: PathBuf, - pub admins: Option, -} - -const DASHBOARD_DIR: &str = "/usr/share/lfsx/dashboard"; - -fn dashboard( - enabled: Option<&str>, - dir: Option<&str>, - repo: Option<&str>, - forge: Option<&str>, - auth: &Auth, - forges: &[Forge], -) -> Option { - if enabled != Some("true") { - return None; - } - - let forge = forge.filter(|forge| !forge.is_empty()); - if let Some(forge) = forge - && !forges.iter().any(|named| named.name == forge) - { - panic!("LFSX_DASHBOARD_FORGE names {forge}, which LFSX_FORGES does not list"); - } - - let admins = repo.filter(|repo| !repo.is_empty()).map(|repo| { - repo.split_once('/') - .and_then(|(org, name)| match forge { - Some(forge) => Namespace::on(forge, org, name).ok(), - None => Namespace::new(org, name).ok(), - }) - .unwrap_or_else(|| panic!("LFSX_DASHBOARD_REPO is not org/repo: {repo}")) - }); - if admins.is_none() && matches!(auth, Auth::Forge { .. }) { - panic!( - "LFSX_DASHBOARD=true needs LFSX_DASHBOARD_REPO: the dashboard is shown to the admins of \ - that repository, and nobody else" - ); - } - - Some(Dashboard { - dir: dir - .filter(|dir| !dir.is_empty()) - .unwrap_or(DASHBOARD_DIR) - .into(), - admins, - }) -} - -#[derive(Debug, Clone)] -pub enum Storage { - Local, - // Endpoint, bucket and credentials all have to be there: a bucket the server - // cannot reach is a server that answers every push with an error, and - // discovering that on the first upload rather than at boot is the wrong - // order. - Bucket { - dialect: Dialect, - // Whether a download is redirected to the bucket instead of streamed - // through this server. Off by default: the streamed path is the one - // that counts bytes, serves ranges and holds the ceiling, and an - // operator should choose to give those up rather than discover it. - presign: bool, - // A local copy of what the bucket holds, so a second reader does not - // pay the round trip again. None is no cache at all, which is what a - // deployment that never set it keeps getting. - cache: Option, - // Whether locks can be taken here. Not read from the environment: it - // starts true and the startup probes turn it off, the same way they turn - // `presign` off, when the store will not prove it can arbitrate between - // two writers racing for the same key. - locking: bool, - }, -} - -#[derive(Debug, Clone)] -pub enum Dialect { - S3 { - endpoint: String, - bucket: String, - region: String, - access_key: String, - secret_key: String, - path_style: bool, - }, - Azure { - endpoint: String, - account: String, - container: String, - credential: AzureCredential, - }, - Gcs { - endpoint: String, - bucket: String, - credential: GcsCredential, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum GcsCredential { - ServiceAccount(PathBuf), - Metadata, - Anonymous, -} - -fn gcs_credential(value: Option<&str>) -> GcsCredential { - match value.filter(|value| !value.is_empty()) { - None => GcsCredential::Metadata, - Some("none") => GcsCredential::Anonymous, - Some(path) => GcsCredential::ServiceAccount(path.into()), - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AzureCredential { - AccountKey(String), - Sas(String), - Identity, -} - -fn azure_credential(key: Option<&str>, sas: Option<&str>) -> AzureCredential { - let key = key.filter(|key| !key.is_empty()); - let sas = sas.filter(|sas| !sas.is_empty()); - - match (key, sas) { - (Some(_), Some(_)) => panic!( - "LFSX_AZURE_ACCOUNT_KEY and LFSX_AZURE_SAS_TOKEN are both set: pick one, or neither \ - to authenticate with the pod's managed or workload identity" - ), - (Some(key), None) => AzureCredential::AccountKey(key.to_owned()), - (None, Some(sas)) => AzureCredential::Sas(sas.to_owned()), - (None, None) => AzureCredential::Identity, - } -} - -impl Storage { - fn from_env() -> Self { - let kind = std::env::var("LFSX_STORAGE").unwrap_or_default(); - if !matches!(kind.as_str(), "s3" | "azure" | "gcs") { - return Self::Local; - } - - let required = |name: &str| { - std::env::var(name) - .ok() - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| panic!("LFSX_STORAGE={kind} needs {name}")) - }; - - let dialect = if kind == "gcs" { - Dialect::Gcs { - endpoint: std::env::var("LFSX_GCS_ENDPOINT") - .ok() - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| "https://storage.googleapis.com".into()), - bucket: required("LFSX_GCS_BUCKET"), - credential: gcs_credential(std::env::var("LFSX_GCS_CREDENTIALS").ok().as_deref()), - } - } else if kind == "azure" { - let account = required("LFSX_AZURE_ACCOUNT"); - Dialect::Azure { - endpoint: std::env::var("LFSX_AZURE_ENDPOINT") - .ok() - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| format!("https://{account}.blob.core.windows.net")), - container: required("LFSX_AZURE_CONTAINER"), - credential: azure_credential( - std::env::var("LFSX_AZURE_ACCOUNT_KEY").ok().as_deref(), - std::env::var("LFSX_AZURE_SAS_TOKEN").ok().as_deref(), - ), - account, - } - } else { - Dialect::S3 { - endpoint: required("LFSX_S3_ENDPOINT"), - bucket: required("LFSX_S3_BUCKET"), - region: std::env::var("LFSX_S3_REGION").unwrap_or_else(|_| "us-east-1".into()), - access_key: required("LFSX_S3_ACCESS_KEY"), - secret_key: required("LFSX_S3_SECRET_KEY"), - path_style: std::env::var("LFSX_S3_PATH_STYLE").as_deref() != Ok("false"), - } - }; - - Self::Bucket { - dialect, - presign: std::env::var("LFSX_S3_PRESIGN").as_deref() == Ok("true"), - cache: disk_cache( - std::env::var("LFSX_S3_CACHE_DIR").ok().as_deref(), - std::env::var("LFSX_S3_CACHE_MAX_BYTES").ok().as_deref(), - ), - locking: true, - } - } -} - -#[derive(Debug, Clone)] -pub enum Auth { - Forge { - provider: Provider, - api_url: String, - // A GitHub App identity for the server's own calls, so the anonymous - // lookup spends the App installation's quota instead of the 60-an-hour - // unauthenticated one. A file path for the key, never the key itself, - // same discipline as the encryption key. - github_app: Option, - cache_ttl: Duration, - rejection_ttl: Duration, - // Lookups a minute this server will spend on the forge, counted only - // when neither cache could answer. None is no ceiling at all. - lookup_budget: Option, - // Whether a request with no credentials is resolved against the forge - // instead of refused. Off unless asked for, because a server that serves - // strangers should be a decision somebody made. - anonymous_read: bool, - // Namespaces whose objects take write access to read, so a repository the - // forge serves publicly can still keep its assets to the people who could - // push them. - restricted: Namespaces, - allowed: Option, - }, - Disabled, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DiskCache { - pub dir: PathBuf, - pub max_bytes: u64, -} - -// A directory and a ceiling, together or not at all. A cache with nowhere to -// live is nothing, and one with no ceiling fills the disk the server also -// stages uploads on, which is a worse outage than the bucket round trips it -// was meant to save. -fn disk_cache(dir: Option<&str>, max_bytes: Option<&str>) -> Option { - let dir = dir.filter(|dir| !dir.is_empty())?; - - let Some(max_bytes) = max_bytes - .map(str::trim) - .and_then(|raw| raw.parse::().ok()) - .filter(|ceiling| *ceiling > 0) - else { - tracing::warn!( - "LFSX_S3_CACHE_DIR is set without a usable LFSX_S3_CACHE_MAX_BYTES, so nothing is \ - cached: a cache with no ceiling would fill the volume this server stages uploads on" - ); - return None; - }; - - Some(DiskCache { - dir: PathBuf::from(dir), - max_bytes, - }) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum KeySource { - File(PathBuf), - Command(String), -} - -// One source or none. Both is a configuration that says two things, and which -// of them the operator trusts with the store is not a guess this server makes. -fn encryption_key(file: Option<&str>, command: Option<&str>) -> Option { - let file = file.filter(|path| !path.is_empty()); - let command = command.filter(|hook| !hook.is_empty()); - - match (file, command) { - (None, None) => None, - (Some(path), None) => Some(KeySource::File(PathBuf::from(path))), - (None, Some(hook)) => Some(KeySource::Command(hook.to_owned())), - (Some(_), Some(_)) => panic!( - "LFSX_ENCRYPTION_KEY_FILE and LFSX_ENCRYPTION_KEY_COMMAND are both set: they are two \ - answers to where the keys live, and picking one for you is how the wrong keys get used" - ), - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct GithubApp { - pub app_id: String, - pub key_file: PathBuf, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Provider { - Github, - Gitlab, - // Gitea and Forgejo, which are one API: Forgejo is a fork of Gitea and - // answers the same routes, so the only thing that tells two instances apart - // is the root they are reached at. - Gitea, -} - -impl Provider { - // None where there is no such thing as the instance. - // - // github.com and gitlab.com are where a repository is unless the operator - // says otherwise, so defaulting there is nearly always right. Gitea is - // software rather than a place. gitea.com exists, but an operator who names - // this provider is almost certainly running their own, and quietly resolving - // their namespaces against a stranger's forge is worse than not starting: a - // public repository there that happens to share a name would hand out - // anonymous read on objects it has nothing to do with. - fn default_api_url(self) -> Option<&'static str> { - match self { - Self::Github => Some("https://api.github.com"), - Self::Gitlab => Some("https://gitlab.com/api/v4"), - Self::Gitea => None, - } - } - - fn api_url_variable(self) -> &'static str { - match self { - Self::Github => "LFSX_GITHUB_API_URL", - Self::Gitlab => "LFSX_GITLAB_API_URL", - Self::Gitea => "LFSX_GITEA_API_URL", - } - } -} - -const CACHE_TTL: Duration = Duration::from_secs(60); -const REJECTION_TTL: Duration = Duration::from_secs(10); -// Generous enough that a busy server never meets it, since a lookup is one -// distinct token against one repository per cache lifetime rather than one per -// request, and tight enough that a flood costs ten a second instead of whatever -// the network will carry. -const LOOKUP_BUDGET: u32 = 600; const TRANSFER_CAP: usize = 128; const GC_GRACE: Duration = Duration::from_secs(14 * 24 * 60 * 60); const STAGING_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); @@ -495,161 +174,6 @@ impl Config { } } -// Opt in, not opt out. Serving objects to a caller with no credentials at all is -// a decision an operator should make on purpose: it costs them the bandwidth of -// anyone who finds the endpoint, on a server whose whole job is to move files -// measured in gigabytes. Nothing confidential is at stake, since a request with -// no credentials is still resolved against the forge and a private repository is -// still refused, but "anyone may pull from you" is not a sensible thing to -// inherit by default. -// -// Only the exact string opens it. A typo, an empty value or a `1` leaves it -// closed, because the failure that matters here is the one that opens the door -// when nobody meant to. -fn anonymous_read(value: Option<&str>) -> bool { - value == Some("true") -} - -impl Auth { - fn from_env() -> Self { - if std::env::var("LFSX_AUTH").as_deref() == Ok("disabled") { - tracing::warn!( - "LFSX_AUTH=disabled: every request is accepted, run this on a trusted network only" - ); - if is_set(std::env::var("LFSX_ALLOWED").ok().as_deref()) { - tracing::warn!( - "LFSX_ALLOWED is set and LFSX_AUTH=disabled, so it does nothing: with no forge \ - to ask, every repository is served" - ); - } - if is_set(std::env::var("LFSX_RESTRICTED").ok().as_deref()) { - tracing::warn!( - "LFSX_RESTRICTED is set and LFSX_AUTH=disabled, so it does nothing: every \ - caller already holds every right" - ); - } - return Self::Disabled; - } - - let provider = provider(std::env::var("LFSX_AUTH").ok().as_deref()); - - Self::Forge { - provider, - api_url: api_url( - provider, - std::env::var(provider.api_url_variable()).ok().as_deref(), - ), - cache_ttl: seconds("LFSX_AUTH_CACHE_TTL").unwrap_or(CACHE_TTL), - rejection_ttl: seconds("LFSX_AUTH_REJECTION_TTL").unwrap_or(REJECTION_TTL), - lookup_budget: lookup_budget(std::env::var("LFSX_AUTH_LOOKUP_BUDGET").ok().as_deref()), - github_app: github_app(provider), - anonymous_read: anonymous_read(std::env::var("LFSX_ANONYMOUS_READ").ok().as_deref()), - restricted: Namespaces::parse( - "LFSX_RESTRICTED", - std::env::var("LFSX_RESTRICTED").ok().as_deref(), - ), - allowed: allowed( - "LFSX_ALLOWED", - std::env::var("LFSX_ALLOWED").ok().as_deref(), - ), - } - } -} - -fn is_set(value: Option<&str>) -> bool { - value.is_some_and(|value| !value.trim().is_empty()) -} - -fn allowed(variable: &str, value: Option<&str>) -> Option { - is_set(value).then(|| Namespaces::parse(variable, value)) -} - -// Both variables or neither. One without the other is a configuration that -// says two things at once, and an operator who set up an App meant to have its -// quota, so the mistake is refused at boot instead of quietly ignored. -fn github_app(provider: Provider) -> Option { - github_app_from(provider, "LFSX_", "LFSX_AUTH", |variable| { - std::env::var(variable).ok() - }) -} - -fn github_app_from( - provider: Provider, - prefix: &str, - auth_variable: &str, - read: impl Fn(&str) -> Option, -) -> Option { - let id_variable = format!("{prefix}GITHUB_APP_ID"); - let key_variable = format!("{prefix}GITHUB_APP_KEY_FILE"); - let id = read(&id_variable).filter(|id| !id.is_empty()); - let key_file = read(&key_variable).filter(|path| !path.is_empty()); - - match (id, key_file) { - (None, None) => None, - (Some(app_id), Some(key_file)) => { - if provider != Provider::Github { - tracing::warn!( - "{id_variable} is set but {auth_variable} is not github, so it does nothing" - ); - return None; - } - Some(GithubApp { - app_id, - key_file: PathBuf::from(key_file), - }) - } - _ => panic!( - "{id_variable} and {key_variable} come together: one without the other is half an \ - identity, and guessing which half was meant is worse than stopping" - ), - } -} - -// Zero is the one value that cannot mean what it says. A ceiling of no lookups -// is a server that refuses every caller it has not already seen, so it is read as -// the operator turning the ceiling off, which is the only other thing they could -// have meant. Anything unparseable is the default rather than a refusal to start: -// this bounds a cost, and getting it wrong should not take the server down. -fn lookup_budget(value: Option<&str>) -> Option { - match value.map(str::trim).map(str::parse::) { - Some(Ok(0)) => None, - Some(Ok(budget)) => Some(budget), - Some(Err(_)) | None => Some(LOOKUP_BUDGET), - } -} - -// Anything unrecognised is GitHub, which is what an operator who set nothing -// almost certainly meant. Forgejo is named alongside Gitea because they are one -// API, and somebody running Forgejo should not have to know it began as a fork. -fn provider(value: Option<&str>) -> Provider { - match value { - Some("gitlab") => Provider::Gitlab, - Some("gitea") | Some("forgejo") => Provider::Gitea, - _ => Provider::Github, - } -} - -// The trailing slash matters: every route is built by appending to this, so one -// left on the end produces `//repos/...`, which some forges answer and others do -// not, and the ones that do not answer 404 for a repository that is right there. -fn api_url(provider: Provider, configured: Option<&str>) -> String { - api_url_from(provider, provider.api_url_variable(), configured) -} - -fn api_url_from(provider: Provider, variable: &str, configured: Option<&str>) -> String { - configured - .map(str::to_owned) - .or_else(|| provider.default_api_url().map(str::to_owned)) - .unwrap_or_else(|| { - panic!( - "{variable} must be set: a self-hosted forge has no default API root, and guessing \ - one would resolve your repositories against somebody else's" - ) - }) - .trim_end_matches('/') - .to_owned() -} - // Is this a host and a port, and nothing else? // // A `Host` carrying a `/` or an `@` is not one, and both change where the URL diff --git a/server/src/config/auth.rs b/server/src/config/auth.rs new file mode 100644 index 0000000..0487b64 --- /dev/null +++ b/server/src/config/auth.rs @@ -0,0 +1,238 @@ +use std::path::PathBuf; +use std::time::Duration; + +use super::seconds; +use crate::auth::Namespaces; + +#[derive(Debug, Clone)] +pub enum Auth { + Forge { + provider: Provider, + api_url: String, + // A GitHub App identity for the server's own calls, so the anonymous + // lookup spends the App installation's quota instead of the 60-an-hour + // unauthenticated one. A file path for the key, never the key itself, + // same discipline as the encryption key. + github_app: Option, + cache_ttl: Duration, + rejection_ttl: Duration, + // Lookups a minute this server will spend on the forge, counted only + // when neither cache could answer. None is no ceiling at all. + lookup_budget: Option, + // Whether a request with no credentials is resolved against the forge + // instead of refused. Off unless asked for, because a server that serves + // strangers should be a decision somebody made. + anonymous_read: bool, + // Namespaces whose objects take write access to read, so a repository the + // forge serves publicly can still keep its assets to the people who could + // push them. + restricted: Namespaces, + allowed: Option, + }, + Disabled, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GithubApp { + pub app_id: String, + pub key_file: PathBuf, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Provider { + Github, + Gitlab, + // Gitea and Forgejo, which are one API: Forgejo is a fork of Gitea and + // answers the same routes, so the only thing that tells two instances apart + // is the root they are reached at. + Gitea, +} + +impl Provider { + // None where there is no such thing as the instance. + // + // github.com and gitlab.com are where a repository is unless the operator + // says otherwise, so defaulting there is nearly always right. Gitea is + // software rather than a place. gitea.com exists, but an operator who names + // this provider is almost certainly running their own, and quietly resolving + // their namespaces against a stranger's forge is worse than not starting: a + // public repository there that happens to share a name would hand out + // anonymous read on objects it has nothing to do with. + fn default_api_url(self) -> Option<&'static str> { + match self { + Self::Github => Some("https://api.github.com"), + Self::Gitlab => Some("https://gitlab.com/api/v4"), + Self::Gitea => None, + } + } + + fn api_url_variable(self) -> &'static str { + match self { + Self::Github => "LFSX_GITHUB_API_URL", + Self::Gitlab => "LFSX_GITLAB_API_URL", + Self::Gitea => "LFSX_GITEA_API_URL", + } + } +} + +const CACHE_TTL: Duration = Duration::from_secs(60); +const REJECTION_TTL: Duration = Duration::from_secs(10); +// Generous enough that a busy server never meets it, since a lookup is one +// distinct token against one repository per cache lifetime rather than one per +// request, and tight enough that a flood costs ten a second instead of whatever +// the network will carry. +pub(super) const LOOKUP_BUDGET: u32 = 600; +// Opt in, not opt out. Serving objects to a caller with no credentials at all is +// a decision an operator should make on purpose: it costs them the bandwidth of +// anyone who finds the endpoint, on a server whose whole job is to move files +// measured in gigabytes. Nothing confidential is at stake, since a request with +// no credentials is still resolved against the forge and a private repository is +// still refused, but "anyone may pull from you" is not a sensible thing to +// inherit by default. +// +// Only the exact string opens it. A typo, an empty value or a `1` leaves it +// closed, because the failure that matters here is the one that opens the door +// when nobody meant to. +pub(super) fn anonymous_read(value: Option<&str>) -> bool { + value == Some("true") +} + +impl Auth { + pub(super) fn from_env() -> Self { + if std::env::var("LFSX_AUTH").as_deref() == Ok("disabled") { + tracing::warn!( + "LFSX_AUTH=disabled: every request is accepted, run this on a trusted network only" + ); + if is_set(std::env::var("LFSX_ALLOWED").ok().as_deref()) { + tracing::warn!( + "LFSX_ALLOWED is set and LFSX_AUTH=disabled, so it does nothing: with no forge \ + to ask, every repository is served" + ); + } + if is_set(std::env::var("LFSX_RESTRICTED").ok().as_deref()) { + tracing::warn!( + "LFSX_RESTRICTED is set and LFSX_AUTH=disabled, so it does nothing: every \ + caller already holds every right" + ); + } + return Self::Disabled; + } + + let provider = provider(std::env::var("LFSX_AUTH").ok().as_deref()); + + Self::Forge { + provider, + api_url: api_url( + provider, + std::env::var(provider.api_url_variable()).ok().as_deref(), + ), + cache_ttl: seconds("LFSX_AUTH_CACHE_TTL").unwrap_or(CACHE_TTL), + rejection_ttl: seconds("LFSX_AUTH_REJECTION_TTL").unwrap_or(REJECTION_TTL), + lookup_budget: lookup_budget(std::env::var("LFSX_AUTH_LOOKUP_BUDGET").ok().as_deref()), + github_app: github_app(provider), + anonymous_read: anonymous_read(std::env::var("LFSX_ANONYMOUS_READ").ok().as_deref()), + restricted: Namespaces::parse( + "LFSX_RESTRICTED", + std::env::var("LFSX_RESTRICTED").ok().as_deref(), + ), + allowed: allowed( + "LFSX_ALLOWED", + std::env::var("LFSX_ALLOWED").ok().as_deref(), + ), + } + } +} + +pub(super) fn is_set(value: Option<&str>) -> bool { + value.is_some_and(|value| !value.trim().is_empty()) +} + +pub(super) fn allowed(variable: &str, value: Option<&str>) -> Option { + is_set(value).then(|| Namespaces::parse(variable, value)) +} + +// Both variables or neither. One without the other is a configuration that +// says two things at once, and an operator who set up an App meant to have its +// quota, so the mistake is refused at boot instead of quietly ignored. +pub(super) fn github_app(provider: Provider) -> Option { + github_app_from(provider, "LFSX_", "LFSX_AUTH", |variable| { + std::env::var(variable).ok() + }) +} + +pub(super) fn github_app_from( + provider: Provider, + prefix: &str, + auth_variable: &str, + read: impl Fn(&str) -> Option, +) -> Option { + let id_variable = format!("{prefix}GITHUB_APP_ID"); + let key_variable = format!("{prefix}GITHUB_APP_KEY_FILE"); + let id = read(&id_variable).filter(|id| !id.is_empty()); + let key_file = read(&key_variable).filter(|path| !path.is_empty()); + + match (id, key_file) { + (None, None) => None, + (Some(app_id), Some(key_file)) => { + if provider != Provider::Github { + tracing::warn!( + "{id_variable} is set but {auth_variable} is not github, so it does nothing" + ); + return None; + } + Some(GithubApp { + app_id, + key_file: PathBuf::from(key_file), + }) + } + _ => panic!( + "{id_variable} and {key_variable} come together: one without the other is half an \ + identity, and guessing which half was meant is worse than stopping" + ), + } +} + +// Zero is the one value that cannot mean what it says. A ceiling of no lookups +// is a server that refuses every caller it has not already seen, so it is read as +// the operator turning the ceiling off, which is the only other thing they could +// have meant. Anything unparseable is the default rather than a refusal to start: +// this bounds a cost, and getting it wrong should not take the server down. +pub(super) fn lookup_budget(value: Option<&str>) -> Option { + match value.map(str::trim).map(str::parse::) { + Some(Ok(0)) => None, + Some(Ok(budget)) => Some(budget), + Some(Err(_)) | None => Some(LOOKUP_BUDGET), + } +} + +// Anything unrecognised is GitHub, which is what an operator who set nothing +// almost certainly meant. Forgejo is named alongside Gitea because they are one +// API, and somebody running Forgejo should not have to know it began as a fork. +pub(super) fn provider(value: Option<&str>) -> Provider { + match value { + Some("gitlab") => Provider::Gitlab, + Some("gitea") | Some("forgejo") => Provider::Gitea, + _ => Provider::Github, + } +} + +// The trailing slash matters: every route is built by appending to this, so one +// left on the end produces `//repos/...`, which some forges answer and others do +// not, and the ones that do not answer 404 for a repository that is right there. +pub(super) fn api_url(provider: Provider, configured: Option<&str>) -> String { + api_url_from(provider, provider.api_url_variable(), configured) +} + +pub(super) fn api_url_from(provider: Provider, variable: &str, configured: Option<&str>) -> String { + configured + .map(str::to_owned) + .or_else(|| provider.default_api_url().map(str::to_owned)) + .unwrap_or_else(|| { + panic!( + "{variable} must be set: a self-hosted forge has no default API root, and guessing \ + one would resolve your repositories against somebody else's" + ) + }) + .trim_end_matches('/') + .to_owned() +} diff --git a/server/src/config/dashboard.rs b/server/src/config/dashboard.rs new file mode 100644 index 0000000..4fa03fb --- /dev/null +++ b/server/src/config/dashboard.rs @@ -0,0 +1,55 @@ +use std::path::PathBuf; + +use super::{Auth, Forge}; +use crate::namespace::Namespace; + +#[derive(Debug, Clone)] +pub struct Dashboard { + pub dir: PathBuf, + pub admins: Option, +} + +pub(super) const DASHBOARD_DIR: &str = "/usr/share/lfsx/dashboard"; + +pub(super) fn dashboard( + enabled: Option<&str>, + dir: Option<&str>, + repo: Option<&str>, + forge: Option<&str>, + auth: &Auth, + forges: &[Forge], +) -> Option { + if enabled != Some("true") { + return None; + } + + let forge = forge.filter(|forge| !forge.is_empty()); + if let Some(forge) = forge + && !forges.iter().any(|named| named.name == forge) + { + panic!("LFSX_DASHBOARD_FORGE names {forge}, which LFSX_FORGES does not list"); + } + + let admins = repo.filter(|repo| !repo.is_empty()).map(|repo| { + repo.split_once('/') + .and_then(|(org, name)| match forge { + Some(forge) => Namespace::on(forge, org, name).ok(), + None => Namespace::new(org, name).ok(), + }) + .unwrap_or_else(|| panic!("LFSX_DASHBOARD_REPO is not org/repo: {repo}")) + }); + if admins.is_none() && matches!(auth, Auth::Forge { .. }) { + panic!( + "LFSX_DASHBOARD=true needs LFSX_DASHBOARD_REPO: the dashboard is shown to the admins of \ + that repository, and nobody else" + ); + } + + Some(Dashboard { + dir: dir + .filter(|dir| !dir.is_empty()) + .unwrap_or(DASHBOARD_DIR) + .into(), + admins, + }) +} diff --git a/server/src/config/forges.rs b/server/src/config/forges.rs index 34ca732..15879cb 100644 --- a/server/src/config/forges.rs +++ b/server/src/config/forges.rs @@ -1,4 +1,4 @@ -use super::{Auth, Provider, allowed, anonymous_read, api_url_from, github_app_from}; +use super::auth::{Auth, Provider, allowed, anonymous_read, api_url_from, github_app_from}; use crate::auth::Namespaces; use crate::namespace::is_forge_name; diff --git a/server/src/config/storage.rs b/server/src/config/storage.rs new file mode 100644 index 0000000..57ca2f8 --- /dev/null +++ b/server/src/config/storage.rs @@ -0,0 +1,201 @@ +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub enum Storage { + Local, + // Endpoint, bucket and credentials all have to be there: a bucket the server + // cannot reach is a server that answers every push with an error, and + // discovering that on the first upload rather than at boot is the wrong + // order. + Bucket { + dialect: Dialect, + // Whether a download is redirected to the bucket instead of streamed + // through this server. Off by default: the streamed path is the one + // that counts bytes, serves ranges and holds the ceiling, and an + // operator should choose to give those up rather than discover it. + presign: bool, + // A local copy of what the bucket holds, so a second reader does not + // pay the round trip again. None is no cache at all, which is what a + // deployment that never set it keeps getting. + cache: Option, + // Whether locks can be taken here. Not read from the environment: it + // starts true and the startup probes turn it off, the same way they turn + // `presign` off, when the store will not prove it can arbitrate between + // two writers racing for the same key. + locking: bool, + }, +} + +#[derive(Debug, Clone)] +pub enum Dialect { + S3 { + endpoint: String, + bucket: String, + region: String, + access_key: String, + secret_key: String, + path_style: bool, + }, + Azure { + endpoint: String, + account: String, + container: String, + credential: AzureCredential, + }, + Gcs { + endpoint: String, + bucket: String, + credential: GcsCredential, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GcsCredential { + ServiceAccount(PathBuf), + Metadata, + Anonymous, +} + +pub(super) fn gcs_credential(value: Option<&str>) -> GcsCredential { + match value.filter(|value| !value.is_empty()) { + None => GcsCredential::Metadata, + Some("none") => GcsCredential::Anonymous, + Some(path) => GcsCredential::ServiceAccount(path.into()), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AzureCredential { + AccountKey(String), + Sas(String), + Identity, +} + +pub(super) fn azure_credential(key: Option<&str>, sas: Option<&str>) -> AzureCredential { + let key = key.filter(|key| !key.is_empty()); + let sas = sas.filter(|sas| !sas.is_empty()); + + match (key, sas) { + (Some(_), Some(_)) => panic!( + "LFSX_AZURE_ACCOUNT_KEY and LFSX_AZURE_SAS_TOKEN are both set: pick one, or neither \ + to authenticate with the pod's managed or workload identity" + ), + (Some(key), None) => AzureCredential::AccountKey(key.to_owned()), + (None, Some(sas)) => AzureCredential::Sas(sas.to_owned()), + (None, None) => AzureCredential::Identity, + } +} + +impl Storage { + pub(super) fn from_env() -> Self { + let kind = std::env::var("LFSX_STORAGE").unwrap_or_default(); + if !matches!(kind.as_str(), "s3" | "azure" | "gcs") { + return Self::Local; + } + + let required = |name: &str| { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| panic!("LFSX_STORAGE={kind} needs {name}")) + }; + + let dialect = if kind == "gcs" { + Dialect::Gcs { + endpoint: std::env::var("LFSX_GCS_ENDPOINT") + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "https://storage.googleapis.com".into()), + bucket: required("LFSX_GCS_BUCKET"), + credential: gcs_credential(std::env::var("LFSX_GCS_CREDENTIALS").ok().as_deref()), + } + } else if kind == "azure" { + let account = required("LFSX_AZURE_ACCOUNT"); + Dialect::Azure { + endpoint: std::env::var("LFSX_AZURE_ENDPOINT") + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| format!("https://{account}.blob.core.windows.net")), + container: required("LFSX_AZURE_CONTAINER"), + credential: azure_credential( + std::env::var("LFSX_AZURE_ACCOUNT_KEY").ok().as_deref(), + std::env::var("LFSX_AZURE_SAS_TOKEN").ok().as_deref(), + ), + account, + } + } else { + Dialect::S3 { + endpoint: required("LFSX_S3_ENDPOINT"), + bucket: required("LFSX_S3_BUCKET"), + region: std::env::var("LFSX_S3_REGION").unwrap_or_else(|_| "us-east-1".into()), + access_key: required("LFSX_S3_ACCESS_KEY"), + secret_key: required("LFSX_S3_SECRET_KEY"), + path_style: std::env::var("LFSX_S3_PATH_STYLE").as_deref() != Ok("false"), + } + }; + + Self::Bucket { + dialect, + presign: std::env::var("LFSX_S3_PRESIGN").as_deref() == Ok("true"), + cache: disk_cache( + std::env::var("LFSX_S3_CACHE_DIR").ok().as_deref(), + std::env::var("LFSX_S3_CACHE_MAX_BYTES").ok().as_deref(), + ), + locking: true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiskCache { + pub dir: PathBuf, + pub max_bytes: u64, +} + +// A directory and a ceiling, together or not at all. A cache with nowhere to +// live is nothing, and one with no ceiling fills the disk the server also +// stages uploads on, which is a worse outage than the bucket round trips it +// was meant to save. +pub(super) fn disk_cache(dir: Option<&str>, max_bytes: Option<&str>) -> Option { + let dir = dir.filter(|dir| !dir.is_empty())?; + + let Some(max_bytes) = max_bytes + .map(str::trim) + .and_then(|raw| raw.parse::().ok()) + .filter(|ceiling| *ceiling > 0) + else { + tracing::warn!( + "LFSX_S3_CACHE_DIR is set without a usable LFSX_S3_CACHE_MAX_BYTES, so nothing is \ + cached: a cache with no ceiling would fill the volume this server stages uploads on" + ); + return None; + }; + + Some(DiskCache { + dir: PathBuf::from(dir), + max_bytes, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum KeySource { + File(PathBuf), + Command(String), +} + +// One source or none. Both is a configuration that says two things, and which +// of them the operator trusts with the store is not a guess this server makes. +pub(super) fn encryption_key(file: Option<&str>, command: Option<&str>) -> Option { + let file = file.filter(|path| !path.is_empty()); + let command = command.filter(|hook| !hook.is_empty()); + + match (file, command) { + (None, None) => None, + (Some(path), None) => Some(KeySource::File(PathBuf::from(path))), + (None, Some(hook)) => Some(KeySource::Command(hook.to_owned())), + (Some(_), Some(_)) => panic!( + "LFSX_ENCRYPTION_KEY_FILE and LFSX_ENCRYPTION_KEY_COMMAND are both set: they are two \ + answers to where the keys live, and picking one for you is how the wrong keys get used" + ), + } +} diff --git a/server/src/config/tests.rs b/server/src/config/tests.rs index 1dfe697..a1ef57e 100644 --- a/server/src/config/tests.rs +++ b/server/src/config/tests.rs @@ -1,4 +1,13 @@ +use std::path::PathBuf; +use std::time::Duration; + +use super::auth::{ + LOOKUP_BUDGET, allowed, anonymous_read, api_url, is_set, lookup_budget, provider, +}; +use super::dashboard::{DASHBOARD_DIR, dashboard}; +use super::storage::{azure_credential, encryption_key, gcs_credential}; use super::*; +use crate::auth::Namespaces; // The default is the whole point of this function: an operator who sets nothing // gets a server that asks for a credential. Everything else here guards against