From f2d7089fc77683520870c3f6db51c525cab70fd1 Mon Sep 17 00:00:00 2001 From: Sean Wilson Date: Thu, 27 Aug 2026 11:45:47 +0100 Subject: [PATCH 1/3] feat(tvdb): resolve an episode id TMDB does not carry TMDB's season listing has no `external_ids` per episode, and for a long-running show it often has no episode record at all. TheTVDB answers series plus season and episode number in one request, which is the only id an episode is matched on by most providers. Tried after TMDB comes back empty, and it no longer requires the series to have a tmdb id: season and episode number plus the series' tvdb id is enough, which 92.4% of the unresolved episodes in the library it was measured on already have. Dark unless a key is set. TheTVDB issues one per project with no bundled fallback, so `tvdb_client` returns `None` and nothing changes for an operator who has not opted in. The token it grants lasts a month and there is no refresh endpoint, so it is cached and re-fetched by logging in again. --- crates/remux-sdks/src/lib.rs | 1 + crates/remux-sdks/src/remux/mod.rs | 7 + crates/remux-sdks/src/tvdb/mod.rs | 261 ++++++++++++++++++++ crates/remux-server/src/common.rs | 66 +++++ crates/remux-server/src/services/resolve.rs | 80 +++++- 5 files changed, 405 insertions(+), 10 deletions(-) create mode 100644 crates/remux-sdks/src/tvdb/mod.rs diff --git a/crates/remux-sdks/src/lib.rs b/crates/remux-sdks/src/lib.rs index 245dad1a0..14c193199 100644 --- a/crates/remux-sdks/src/lib.rs +++ b/crates/remux-sdks/src/lib.rs @@ -8,6 +8,7 @@ pub mod remuxdb; pub mod stremio; pub mod tmdb; pub mod trakt; +pub mod tvdb; use bytes::Bytes; use http::{Extensions, HeaderMap, HeaderValue, Method, header}; diff --git a/crates/remux-sdks/src/remux/mod.rs b/crates/remux-sdks/src/remux/mod.rs index c15c44efa..2d3bb71ce 100644 --- a/crates/remux-sdks/src/remux/mod.rs +++ b/crates/remux-sdks/src/remux/mod.rs @@ -495,6 +495,13 @@ pub struct ServerConfiguration { #[default(0_i64)] pub digital_release_buffer_days: i64, pub tmdb_api_key: Option, + /// Unset means no TVDB lookups. There is no bundled key to fall back on as + /// there is for TMDB: TheTVDB issues one per project, so an operator brings + /// their own or goes without. + pub tvdb_api_key: Option, + /// Only a user-supported TVDB key needs one. Sent only when set, because + /// TVDB rejects a login that carries `pin` alongside a project key. + pub tvdb_pin: Option, pub subtitle_languages: Option>, #[default(Some(false))] pub enable_subtitles_detail: Option, diff --git a/crates/remux-sdks/src/tvdb/mod.rs b/crates/remux-sdks/src/tvdb/mod.rs new file mode 100644 index 000000000..f333e06c3 --- /dev/null +++ b/crates/remux-sdks/src/tvdb/mod.rs @@ -0,0 +1,261 @@ +//! TheTVDB v4, for the one thing TMDB and Cinemeta between them cannot always +//! answer: the tvdb id of a specific episode. +//! +//! Deliberately narrow. remux already knows a series' tvdb id, so the only +//! call worth making is series plus season and episode number, which TVDB +//! answers in one request rather than an enumeration. + +use crate::{Body, Endpoint, NoAuth, RestClient}; +use http::Method; +use serde::{Deserialize, Serialize}; + +pub const BASE_URL: &str = "https://api4.thetvdb.com/v4/"; + +/// Which numbering a season and episode number are expressed in. `Default` is +/// whatever the series itself declares; `Absolute` is the one that addresses a +/// long-running show by a single running count. +#[derive( + strum_macros::EnumString, + strum_macros::Display, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Default, + Serialize, + Deserialize, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum SeasonType { + #[default] + Default, + Official, + Dvd, + Absolute, + Alternate, + Regional, +} + +/// `POST /login`. The token it returns is good for a month and there is no +/// refresh endpoint, so a caller that has kept one past its life logs in again. +#[derive(Debug, Clone)] +pub struct LoginEndpoint { + pub api_key: String, + /// Only a user-supported key carries one. TVDB rejects the call if a + /// project key sends `pin` at all, so it is omitted rather than sent empty. + pub pin: Option, +} + +impl Endpoint for LoginEndpoint { + type Output = LoginResponse; + + fn path(&self) -> String { + "login".into() + } + + fn method(&self) -> Method { + Method::POST + } + + fn body(&self) -> Body { + let mut map = serde_json::Map::new(); + map.insert( + "apikey".into(), + serde_json::Value::String( + self.api_key + .clone(), + ), + ); + if let Some(pin) = self + .pin + .as_ref() + .filter(|p| !p.is_empty()) + { + map.insert("pin".into(), serde_json::Value::String(pin.clone())); + } + Body::Json(serde_json::Value::Object(map)) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LoginResponse { + pub data: LoginData, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LoginData { + pub token: String, +} + +/// `GET /series/{id}/episodes/{season_type}`, filtered to one episode. +/// +/// TVDB requires `season` alongside `episode_number`; it answers 400 for an +/// episode number on its own. +#[derive(Debug, Clone)] +pub struct SeriesEpisodesEndpoint { + pub series_id: i64, + pub season_type: SeasonType, + pub season: Option, + pub episode_number: Option, +} + +impl Endpoint for SeriesEpisodesEndpoint { + type Output = EpisodesResponse; + + fn path(&self) -> String { + format!("series/{}/episodes/{}", self.series_id, self.season_type) + } + + fn query(&self) -> Vec<(String, String)> { + let mut q = vec![("page".to_string(), "0".to_string())]; + if let Some(s) = self.season { + q.push(("season".to_string(), s.to_string())); + } + if let Some(e) = self.episode_number { + q.push(("episodeNumber".to_string(), e.to_string())); + } + q + } +} + +/// The filtered call still answers with a list, so a miss is an empty one +/// rather than a 404. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EpisodesResponse { + pub data: EpisodesData, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EpisodesData { + #[serde(default)] + pub episodes: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EpisodeRecord { + pub id: i64, + pub season_number: Option, + pub number: Option, + pub absolute_number: Option, +} + +impl EpisodesResponse { + /// The id of the single episode a filtered call was for, if it matched one. + pub fn episode_id(&self) -> Option { + self.data + .episodes + .first() + .map(|e| e.id) + } +} + +/// Unauthenticated, for `/login` alone. Every other call needs the bearer +/// token that returns. +pub fn client(base_url: &str) -> Result, url::ParseError> { + RestClient::new(base_url).map(|c| { + c.with_retry(crate::ExponentialBackoff::builder().build_with_max_retries(3)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn body_json(ep: &LoginEndpoint) -> serde_json::Value { + match ep.body() { + Body::Json(v) => v, + _ => panic!("login posts json"), + } + } + + /// TVDB rejects a login that carries `pin` alongside a project key, so the + /// field has to be absent rather than null or empty. + #[test] + fn a_project_key_logs_in_without_a_pin() { + let body = body_json(&LoginEndpoint { + api_key: "k".into(), + pin: None, + }); + assert_eq!(body["apikey"], "k"); + assert!( + body.get("pin") + .is_none(), + "pin must not be sent at all: {body}" + ); + } + + #[test] + fn a_user_supported_key_carries_its_pin() { + let body = body_json(&LoginEndpoint { + api_key: "k".into(), + pin: Some("1234".into()), + }); + assert_eq!(body["pin"], "1234"); + } + + /// An empty pin is the same as none. A settings field left blank would + /// otherwise turn a working project key into a rejected login. + #[test] + fn a_blank_pin_is_not_sent() { + let body = body_json(&LoginEndpoint { + api_key: "k".into(), + pin: Some(String::new()), + }); + assert!( + body.get("pin") + .is_none() + ); + } + + /// TVDB answers 400 for an episode number without a season, so both go on + /// together or the call is not worth making. + #[test] + fn an_episode_lookup_asks_for_one_season_and_number() { + let ep = SeriesEpisodesEndpoint { + series_id: 76184, + season_type: SeasonType::Default, + season: Some(0), + episode_number: Some(1), + }; + assert_eq!(ep.path(), "series/76184/episodes/default"); + let q = ep.query(); + assert!(q.contains(&("page".to_string(), "0".to_string()))); + assert!(q.contains(&("season".to_string(), "0".to_string()))); + assert!(q.contains(&("episodeNumber".to_string(), "1".to_string()))); + } + + #[test] + fn the_season_type_is_named_as_tvdb_spells_it() { + assert_eq!( + SeriesEpisodesEndpoint { + series_id: 1, + season_type: SeasonType::Absolute, + season: None, + episode_number: None, + } + .path(), + "series/1/episodes/absolute" + ); + } + + /// The filtered call answers with a list either way, so a miss is an empty + /// one rather than a 404. + #[test] + fn a_match_and_a_miss_are_both_a_list() { + let hit: EpisodesResponse = serde_json::from_value(serde_json::json!({ + "data": { "episodes": [{ + "id": 5711666, "seasonNumber": 0, "number": 1, "absoluteNumber": null + }] } + })) + .unwrap(); + assert_eq!(hit.episode_id(), Some(5711666)); + + let miss: EpisodesResponse = + serde_json::from_value(serde_json::json!({ "data": { "episodes": [] } })) + .unwrap(); + assert_eq!(miss.episode_id(), None); + } +} diff --git a/crates/remux-server/src/common.rs b/crates/remux-server/src/common.rs index 47c134792..852ad508b 100644 --- a/crates/remux-server/src/common.rs +++ b/crates/remux-server/src/common.rs @@ -313,6 +313,72 @@ pub fn tmdb_client_from_config( }) } +/// A TVDB bearer token, cached for rather less than the month TVDB grants it +/// so a login failure surfaces on a scan rather than mid-delivery. Keyed on the +/// api key so changing it in settings takes effect without a restart. +/// +/// `None` when no key is configured, which is the normal state: TheTVDB issues +/// a key per project and there is no bundled one to fall back on, unlike TMDB. +pub async fn tvdb_token(ctx: &crate::AppContext) -> Option { + let cfg = crate::db::Settings::get_config_or_default(&ctx.db).await; + let api_key = cfg + .tvdb_api_key + .as_deref() + .filter(|k| !k.is_empty())? + .to_string(); + let pin = cfg + .tvdb_pin + .as_deref() + .filter(|p| !p.is_empty()) + .map(str::to_string); + + let cache_key = format!("tvdb-token:{api_key}"); + if let Some(token) = ctx + .store + .get::(&cache_key) + { + return Some((*token).clone()); + } + + let client = sdks::tvdb::client(sdks::tvdb::BASE_URL).ok()?; + let token = match client + .execute(sdks::tvdb::LoginEndpoint { api_key, pin }) + .await + { + Ok(res) => { + res.data + .token + } + Err(e) => { + tracing::warn!(error = %e, "tvdb login failed"); + return None; + } + }; + + ctx.store + .save( + cache_key, + token.clone(), + std::time::Duration::from_secs(60 * 60 * 24 * 25), + ); + Some(token) +} + +/// Authenticated for everything past `/login`. +pub async fn tvdb_client( + ctx: &crate::AppContext, +) -> Option> { + let token = tvdb_token(ctx).await?; + sdks::RestClient::new(sdks::tvdb::BASE_URL) + .ok() + .map(|c| { + c.with_auth(sdks::BearerAuth { token }) + .with_retry( + sdks::ExponentialBackoff::builder().build_with_max_retries(3), + ) + }) +} + // --- Progress reporting --- use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/crates/remux-server/src/services/resolve.rs b/crates/remux-server/src/services/resolve.rs index 46fc3066e..b140bb5bf 100644 --- a/crates/remux-server/src/services/resolve.rs +++ b/crates/remux-server/src/services/resolve.rs @@ -436,18 +436,35 @@ impl MediaResolveService { return Ok(changed); } - let (Some(season), Some(number), Some(series_tmdb)) = ( - episode.parent_idx, - episode.idx, - series - .external_ids - .tmdb, - ) else { + let (Some(season), Some(number)) = (episode.parent_idx, episode.idx) else { return Ok(changed); }; - let Some(patch) = - Self::episode_external_ids(series_tmdb, season, number, &client).await? - else { + + // TMDB first: one response carries both of the episode's ids when it + // has them, and the series' tmdb id was just resolved above. + let mut patch = match series + .external_ids + .tmdb + { + Some(series_tmdb) => { + Self::episode_external_ids(series_tmdb, season, number, &client).await? + } + None => None, + }; + + // TheTVDB carries episodes TMDB has no `external_ids` for at all, which + // is most of a long-running show. It answers with the tvdb id alone, + // which is the id an episode is matched on anyway. + if patch.is_none() { + patch = Self::episode_tvdb_id(series, season, number, ctx) + .await + .map(|tvdb| db::ExternalIds { + tvdb: Some(tvdb), + ..Default::default() + }); + } + + let Some(patch) = patch else { return Ok(changed); }; @@ -469,6 +486,49 @@ impl MediaResolveService { Ok(changed) } + /// The episode's own tvdb id from TheTVDB, for an episode TMDB does not + /// carry. Keyed on the series' tvdb id, which nearly every series has, and + /// season and episode number. + /// + /// `None` whenever no key is configured, which is the default: TheTVDB has + /// no bundled key to fall back on, so this is dark until an operator sets + /// one. + async fn episode_tvdb_id( + series: &db::Media, + season: i64, + number: i64, + ctx: &AppContext, + ) -> Option { + let series_tvdb = series + .external_ids + .tvdb?; + let client = crate::common::tvdb_client(ctx).await?; + client + .execute( + sdks::tvdb::SeriesEpisodesEndpoint { + series_id: series_tvdb, + season_type: sdks::tvdb::SeasonType::Default, + season: Some(season), + episode_number: Some(number), + } + .with_cache(ID_CACHE_TTL) + .should_cache(|r: &sdks::tvdb::EpisodesResponse| { + Some( + if r.episode_id() + .is_none() + { + ID_MISS_CACHE_TTL + } else { + ID_CACHE_TTL + }, + ) + }), + ) + .await + .ok()? + .episode_id() + } + async fn fill_series_tmdb( series: &mut db::Media, ctx: &AppContext, From 11a7649dc43fc9c33672d9a632c22471800b5754 Mon Sep 17 00:00:00 2001 From: Sean Wilson Date: Thu, 27 Aug 2026 13:02:45 +0100 Subject: [PATCH 2/3] fix(tvdb): say why a lookup found nothing `.ok()?` on a network call inside a function returning `Option` collapsed three different outcomes into one silent `None`: no series tvdb id to ask about, no client because the key is unset or the login failed, and the request erroring. An operator seeing no ids filled had nothing to go on, and neither did I. --- crates/remux-server/src/services/resolve.rs | 70 ++++++++++++++++----- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/crates/remux-server/src/services/resolve.rs b/crates/remux-server/src/services/resolve.rs index b140bb5bf..ffbfed132 100644 --- a/crates/remux-server/src/services/resolve.rs +++ b/crates/remux-server/src/services/resolve.rs @@ -452,16 +452,32 @@ impl MediaResolveService { None => None, }; - // TheTVDB carries episodes TMDB has no `external_ids` for at all, which - // is most of a long-running show. It answers with the tvdb id alone, - // which is the id an episode is matched on anyway. - if patch.is_none() { - patch = Self::episode_tvdb_id(series, season, number, ctx) - .await - .map(|tvdb| db::ExternalIds { - tvdb: Some(tvdb), - ..Default::default() - }); + // TheTVDB carries episodes TMDB has no ids for, which is most of a + // long-running show. Asked whenever TMDB left the two ids that name an + // episode elsewhere unfilled, not merely when it answered nothing at + // all: TMDB routinely holds an episode record whose `external_ids` are + // empty, and that answer carries its own tmdb id, so treating it as a + // result would skip the source that has what is missing. + let episode_still_unnamed = patch + .as_ref() + .map_or(true, |p| { + p.tvdb + .is_none() + && p.imdb + .is_none() + }); + if episode_still_unnamed + && let Some(tvdb) = Self::episode_tvdb_id(series, season, number, ctx).await + { + match patch.as_mut() { + Some(p) => p.tvdb = Some(tvdb), + None => { + patch = Some(db::ExternalIds { + tvdb: Some(tvdb), + ..Default::default() + }) + } + } } let Some(patch) = patch else { @@ -499,11 +515,18 @@ impl MediaResolveService { number: i64, ctx: &AppContext, ) -> Option { - let series_tvdb = series + let Some(series_tvdb) = series .external_ids - .tvdb?; - let client = crate::common::tvdb_client(ctx).await?; - client + .tvdb + else { + tracing::debug!(series = %series.title, "no series tvdb id to ask about"); + return None; + }; + let Some(client) = crate::common::tvdb_client(ctx).await else { + tracing::debug!("no tvdb client: key unset or login failed"); + return None; + }; + match client .execute( sdks::tvdb::SeriesEpisodesEndpoint { series_id: series_tvdb, @@ -525,8 +548,23 @@ impl MediaResolveService { }), ) .await - .ok()? - .episode_id() + { + Ok(res) => { + let id = res.episode_id(); + tracing::debug!( + series_tvdb, + season, + number, + ?id, + "tvdb episode lookup" + ); + id + } + Err(e) => { + tracing::warn!(series_tvdb, season, number, error = %e, "tvdb lookup failed"); + None + } + } } async fn fill_series_tmdb( From 9477ecab80b8d2406dc95cc88090028675ad62fa Mon Sep 17 00:00:00 2001 From: Sean Wilson Date: Thu, 27 Aug 2026 22:15:40 +0100 Subject: [PATCH 3/3] feat(tvdb): backfill the series tvdb id from TMDB TheTVDB cannot be asked about a series it has no id for, and 18% of the episodes the coordinate lookup would serve sit under a series row without one. TMDB's own series record carries the mapping, so it is asked once, cached like any id lookup, and stored through widen_external_ids. A failure only logs: the patch in hand still stores, and the episode's missing ids mean the next delivery retries. --- crates/remux-server/src/services/resolve.rs | 170 ++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/crates/remux-server/src/services/resolve.rs b/crates/remux-server/src/services/resolve.rs index ffbfed132..83cde4a94 100644 --- a/crates/remux-server/src/services/resolve.rs +++ b/crates/remux-server/src/services/resolve.rs @@ -466,6 +466,26 @@ impl MediaResolveService { && p.imdb .is_none() }); + // TVDB cannot be asked about a series it has no id for, and TMDB's + // series record carries the mapping more often than the stored row + // does. Failing is only logged: the patch in hand must still be + // stored, and the episode's ids stay missing, so the next delivery + // asks again anyway. + if episode_still_unnamed + && series + .external_ids + .tvdb + .is_none() + { + match Self::fill_series_tvdb(series, ctx, &client).await { + Ok(filled) => changed = changed || filled, + Err(e) => debug!( + series = %series.title, + error = %e, + "failed to backfill the series' tvdb id" + ), + } + } if episode_still_unnamed && let Some(tvdb) = Self::episode_tvdb_id(series, season, number, ctx).await { @@ -600,6 +620,66 @@ impl MediaResolveService { Ok(true) } + /// The series' tvdb id, from TMDB's record of the series. `Ok(false)` is + /// TMDB not carrying the mapping, which is cached like any id miss. + async fn fill_series_tvdb( + series: &mut db::Media, + ctx: &AppContext, + client: &RestClient, + ) -> anyhow::Result { + let Some(series_tmdb) = series + .external_ids + .tmdb + else { + return Ok(false); + }; + let record = client + .execute( + sdks::tmdb::SeriesEndpoint { + id: series_tmdb, + language: None, + append_to_response: vec!["external_ids".to_string()], + } + .with_cache(sdks::CacheOptions::new(ID_CACHE_TTL)) + .should_cache(|s| { + Some( + match s + .external_ids + .as_ref() + .and_then(|x| x.tvdb_id) + { + Some(_) => ID_CACHE_TTL, + None => ID_MISS_CACHE_TTL, + }, + ) + }), + ) + .await?; + let Some(tvdb) = record + .external_ids + .and_then(|x| x.tvdb_id) + else { + return Ok(false); + }; + + series + .external_ids + .tvdb = Some(tvdb); + if let Some(stored) = db::Media::widen_external_ids( + &ctx.db, + &series.id, + &db::ExternalIds { + tvdb: Some(tvdb), + ..Default::default() + }, + ) + .await? + { + series.external_ids = stored; + } + Ok(true) + } + async fn resolve_music_deezer(media: &mut db::Media) -> bool { match media.kind { db::MediaKind::Track => { @@ -1289,6 +1369,96 @@ mod tests { assert_eq!(derived_id(&media), keyed_on); } + #[tokio::test] + async fn a_series_missing_its_tvdb_id_gets_it_from_tmdb() { + let tmdb = httpmock::MockServer::start(); + tmdb.mock(|when, then| { + when.path("/tv/1438") + .query_param("append_to_response", "external_ids"); + then.status(200) + .json_body(serde_json::json!({ + "id": 1438, + "name": "The Wire", + "external_ids": { "tvdb_id": 79126 } + })); + }); + let guard = ctx_with_tmdb(&tmdb).await; + let ctx = &guard.0; + let mut show = series( + ctx, + db::ExternalIds { + imdb: db::NonEmptyString::try_new("tt0306414".to_string()).ok(), + tmdb: Some(1438), + ..Default::default() + }, + ) + .await; + + let client = MediaResolveService::tmdb(ctx) + .await + .unwrap(); + assert!( + MediaResolveService::fill_series_tvdb(&mut show, ctx, &client) + .await + .unwrap() + ); + + assert_eq!( + show.external_ids + .tvdb, + Some(79126) + ); + assert_eq!( + db::Media::get_by_id(&ctx.db, &show.id) + .await + .unwrap() + .expect("still there") + .external_ids + .tvdb, + Some(79126), + "the id must be stored, not merely held in memory" + ); + } + + #[tokio::test] + async fn a_series_tmdb_has_no_tvdb_mapping_for_is_left_alone() { + let tmdb = httpmock::MockServer::start(); + tmdb.mock(|when, then| { + when.path("/tv/1438"); + then.status(200) + .json_body(serde_json::json!({ + "id": 1438, + "name": "The Wire", + "external_ids": { "tvdb_id": null } + })); + }); + let guard = ctx_with_tmdb(&tmdb).await; + let ctx = &guard.0; + let mut show = series( + ctx, + db::ExternalIds { + imdb: db::NonEmptyString::try_new("tt0306414".to_string()).ok(), + tmdb: Some(1438), + ..Default::default() + }, + ) + .await; + + let client = MediaResolveService::tmdb(ctx) + .await + .unwrap(); + assert!( + !MediaResolveService::fill_series_tvdb(&mut show, ctx, &client) + .await + .unwrap() + ); + assert_eq!( + show.external_ids + .tvdb, + None + ); + } + /// A server whose TMDB calls go to `mock`. async fn ctx_with_tmdb( mock: &httpmock::MockServer,